Merge branch 'master' of https://github.com/Future-Outlier/flyte into…

… agent-metadata-proto-service
flyteorg · Dec 10, 2023 · d764139 · d764139
2 parents 8f28ffd + 91d24a9
commit d764139
Show file tree

Hide file tree

Showing 176 changed files with 47,658 additions and 614 deletions.
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -118,6 +118,8 @@ jobs:
     with:
       component: ${{ matrix.component }}
       go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
 
   build_docker_images:
     name: Build Images

diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml
@@ -25,4 +25,4 @@ jobs:
       - name: Codespell
         uses: codespell-project/actions-codespell@v2
         with:
-          skip: "*.pb"
+          skip: "*.pb,monodocs-environment.lock.yaml"
diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml
@@ -79,25 +79,16 @@ jobs:
       - name: Register P0 tests
         if: ${{ inputs.priorities == 'P0' }}
         run: |
-          for f in \
-                  basics/basics/hello_world.py \
-                  basics/basics/workflow.py \
-                  basics/basics/named_outputs.py \
-                  advanced_composition/advanced_composition/chain_entities.py \
-                  advanced_composition/advanced_composition/dynamics.py \
-                  advanced_composition/advanced_composition/map_task.py \
-                  advanced_composition/advanced_composition/subworkflows.py \
-                  data_types_and_io/data_types_and_io/dataclass.py \
-                  data_types_and_io/data_types_and_io/structured_dataset.py ;
+          while read -r line;
           do
               pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
                   register \
                   --project flytesnacks \
                   --domain development \
                   --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \
                   --version ${{ env.FLYTESNACKS_VERSION }} \
-                  flytesnacks/examples/$f;
-          done
+                  flytesnacks/$line;
+          done < flytesnacks/flyte_tests.txt
       - name: Register all flytesnacks examples
         if: ${{ inputs.priorities != 'P0' }}
         uses: unionai/[email protected]

diff --git a/.github/workflows/go_generate.yml b/.github/workflows/go_generate.yml
@@ -9,6 +9,9 @@ on:
       go-version:
         required: true
         type: string
+    secrets:
+      FLYTE_BOT_PAT:
+        required: true
 jobs:
   generate:
     runs-on: ubuntu-latest
@@ -19,6 +22,8 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: arduino/setup-protoc@v1
+        with:
+          repo-token: ${{ secrets.FLYTE_BOT_PAT }}
       - uses: bufbuild/buf-setup-action@v1
       - uses: actions/setup-go@v3
         with:

diff --git a/.github/workflows/single-binary.yml b/.github/workflows/single-binary.yml
@@ -27,7 +27,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.19'
+          go-version: "1.19"
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
@@ -129,8 +129,8 @@ jobs:
   sandbox-bundled-functional-tests:
     runs-on: ubuntu-latest
     env:
-      FLYTESNACKS_PRIORITIES: 'P0'
-      FLYTESNACKS_VERSION: ''
+      FLYTESNACKS_PRIORITIES: "P0"
+      FLYTESNACKS_VERSION: ""
     timeout-minutes: 20
     needs: [build-and-push-single-binary-image]
     steps:
@@ -164,7 +164,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.11'
+          python-version: "3.11"
       - uses: unionai/[email protected]
       - name: Setup sandbox
         run: |
@@ -193,25 +193,16 @@ jobs:
           # ref: ${{ env.FLYTESNACKS_VERSION }}
       - name: Register specific tests
         run: |
-          for f in \
-                  basics/basics/hello_world.py \
-                  basics/basics/workflow.py \
-                  basics/basics/named_outputs.py \
-                  advanced_composition/advanced_composition/chain_entities.py \
-                  advanced_composition/advanced_composition/dynamics.py \
-                  advanced_composition/advanced_composition/map_task.py \
-                  advanced_composition/advanced_composition/subworkflows.py \
-                  data_types_and_io/data_types_and_io/dataclass.py \
-                  data_types_and_io/data_types_and_io/structured_dataset.py ;
+          while read -r line;
           do
               pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
                   register \
                   --project flytesnacks \
                   --domain development \
                   --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \
                   --version ${{ env.FLYTESNACKS_VERSION }} \
-                  flytesnacks/examples/$f;
-          done
+                  flytesnacks/$line;
+          done < flytesnacks/flyte_tests.txt
       - name: End2End
         run: |
           make end2end_execute

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -28,17 +28,30 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Fetch the code
-        uses: actions/checkout@v2
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+        uses: actions/checkout@v4
+      - uses: conda-incubator/setup-miniconda@v3
         with:
-          python-version: "3.8"
-      - name: Install dependencies
+          auto-update-conda: true
+          python-version: 3.9
+      - shell: bash -el {0}
+        run: |
+          conda install -c conda-forge conda-lock
+          conda-lock install -n monodocs-env monodocs-environment.lock.yaml
+      - shell: bash -el {0}
         run: |
-          python -m pip install --upgrade pip
-          if [ -f doc-requirements.txt ]; then pip install -r doc-requirements.txt; fi
+          conda activate monodocs-env
+          conda info
+          conda list
+          conda config --show-sources
+          conda config --show
+          printenv | sort
+      - name: Setup Graphviz
+        uses: ts-graphviz/setup-graphviz@v1
       - name: Build the documentation
-        run: make docs
+        shell: bash -el {0}
+        run: |
+          conda activate monodocs-env
+          make docs
 
   generate_kustomize:
     runs-on: ubuntu-latest

diff --git a/.gitignore b/.gitignore
@@ -10,10 +10,6 @@ _repos/
 _rsts/
 rsts_tmp/
 .doctrees/
-docs/_sources/
-docs/flytekit/flytekit.interfaces.html
-docs/searchindex.js
-docs/
 !flyteidl/protos/docs
 release/
 __pycache__/
@@ -33,5 +29,14 @@ dist
 *.db
 vendor/
 /docker/sandbox-bundled/images/tar
-rsts/_tags/
-**/bin/
+**/bin/
+docs/_tags/
+docs/flytectl
+docs/protos
+docs/flytekit
+docs/flytesnacks
+docs/examples
+docs/_src
+docs/_projects
+docs/api
+docs/tests
diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -16,4 +16,4 @@ sphinx:
 # Optionally set the version of Python and requirements required to build your docs
 python:
   install:
-    - requirements: doc-requirements.txt
+    - requirements: doc-requirements.in
diff --git a/CHANGELOG/CHANGELOG-v0.13.0.md b/CHANGELOG/CHANGELOG-v0.13.0.md
@@ -2,8 +2,8 @@
 
 ## Platform
 - Oauth2 support with SingleSignOn and configuration examples for popular IDP's now available in Flyte.
-  Please see the updated [information and description of the feature](https://github.com/flyteorg/flyte/blob/master/rsts/howto/authentication/index.rst), and the [setup information](https://github.com/flyteorg/flyte/blob/master/rsts/howto/authentication/setup.rst)
-  **Attention: If using Auth already - this is a BREAKING change**. refer to the [migration guide](https://github.com/flyteorg/flyte/blob/master/rsts/howto/authentication/migration.rst) to update configuration to ensure Admin continues to work. (No migration needed if auth is not turned on.)
+  Please see the updated [information and description of the feature](https://github.com/flyteorg/flyte/blob/master/docs/howto/authentication/index.rst), and the [setup information](https://github.com/flyteorg/flyte/blob/master/docs/howto/authentication/setup.rst)
+  **Attention: If using Auth already - this is a BREAKING change**. refer to the [migration guide](https://github.com/flyteorg/flyte/blob/master/docs/howto/authentication/migration.rst) to update configuration to ensure Admin continues to work. (No migration needed if auth is not turned on.)
 
 * Backend improvements to support dynamic workflow visualization (in future releases).
 * Lot of features added to [flytectl](https://flytectl.readthedocs.io/en/latest/) .

diff --git a/CHANGELOG/CHANGELOG-v1.10.6.md b/CHANGELOG/CHANGELOG-v1.10.6.md
@@ -0,0 +1,157 @@
+# Flyte 1.10.6 Release
+
+Due to a mishap in the move to the monorepo, we ended up generating the git tags between 1.10.1 to 1.10.5, so in order to decrease the confusion we decided to skip those patch versions and go straight to the next available version.
+
+We've shipped a ton of stuff in this patch release, here are some of the highlights.
+
+## GPU Accelerators
+You'll be able to get more fine-grained in the use GPU Accelerators in your tasks. Here are some examples:
+
+No preference of GPU accelerator to use:
+```
+@task(limits=Resources(gpu="1"))
+def my_task() -> None:
+    ...
+```
+
+Schedule on a specific GPU accelerator:
+```
+from flytekit.extras.accelerators import T4
+
+
+@task(
+    limits=Resources(gpu="1"),
+    accelerator=T4,
+)
+def my_task() -> None:
+    ...
+```
+
+Schedule on a Multi-instance GPU (MIG) accelerator with no preference of partition size:
+```
+from flytekit.extras.accelerators import A100
+
+
+@task(
+    limits=Resources(gpu="1"),
+    accelerator=A100,
+)
+def my_task() -> None:
+    ...
+```
+
+Schedule on a Multi-instance GPU (MIG) accelerator with a specific partition size:
+```
+from flytekit.extras.accelerators import A100
+
+
+@task(
+    limits=Resources(gpu="1"),
+    accelerator=A100.partition_1g_5gb,
+)
+def my_task() -> None:
+    ...
+```
+
+Schedule on an unpartitioned Multi-instance GPU (MIG) accelerator:
+```
+from flytekit.extras.accelerators import A100
+
+
+@task(
+    limits=Resources(gpu="1"),
+    accelerator=A100.unpartitioned,
+)
+def my_task() -> None:
+    ...
+```
+
+## Improved support for Ray logs
+https://github.com/flyteorg/flyte/pull/4266 opens the door for RayJob logs to be persisted.
+
+In https://github.com/flyteorg/flyte/pull/4397 we added support for a link to a Ray dashboard to show up in the task card.
+
+## Updated grafana dashboards
+We updated the official grafana dashboards in https://github.com/flyteorg/flyte/pull/4382.
+
+## Support for Azure AD
+A new version of our stow fork added support for Azure AD in https://github.com/flyteorg/stow/pull/9.
+
+## Full changelog:
+* Restructure Flyte releases by @eapolinario in https://github.com/flyteorg/flyte/pull/4304
+* Use debian bookworm as single binary base image  by @eapolinario in https://github.com/flyteorg/flyte/pull/4311
+* Use local version in single-binary by @eapolinario in https://github.com/flyteorg/flyte/pull/4294
+* Accessibility for README by @mishmanners in https://github.com/flyteorg/flyte/pull/4322
+* Add tests in `flytepropeller/pkg /controller/executors`  from 72.3% to 87.3% coverage by @Future-Outlier in https://github.com/flyteorg/flyte/pull/4276
+* fix: remove unused setting in deployment charts by @HeetVekariya in https://github.com/flyteorg/flyte/pull/4252
+* Document simplified retry behaviour introduced in #3902 by @fg91 in https://github.com/flyteorg/flyte/pull/4022
+* Ray logs persistence by @jeevb in https://github.com/flyteorg/flyte/pull/4266
+* Not revisiting task nodes and correctly incrementing parallelism by @hamersaw in https://github.com/flyteorg/flyte/pull/4318
+* Fix RunPluginEndToEndTest util by @andresgomezfrr in https://github.com/flyteorg/flyte/pull/4342
+* Tune sandbox readiness checks to ensure that sandbox is fully accessi… by @jeevb in https://github.com/flyteorg/flyte/pull/4348
+* Chore: Ensure Stalebot doesn't close issues we've not yet triaged. by @brndnblck in https://github.com/flyteorg/flyte/pull/4352
+* Do not automatically close stale issues by @eapolinario in https://github.com/flyteorg/flyte/pull/4353
+* Fix: Set flyteadmin gRPC port to 80 in ingress if using TLS between load balancer and backend by @fg91 in https://github.com/flyteorg/flyte/pull/3964
+* Support Databricks WebAPI 2.1 version and Support `existing_cluster_id` and `new_cluster` options to create a Job by @Future-Outlier in https://github.com/flyteorg/flyte/pull/4361
+* Fixing caching on maptasks when using partials by @hamersaw in https://github.com/flyteorg/flyte/pull/4344
+* Fix read raw limit by @honnix in https://github.com/flyteorg/flyte/pull/4370
+* minor fix to eks-starter.yaml by @guyarad in https://github.com/flyteorg/flyte/pull/4337
+* Reporting running if the primary container status is not yet reported by @hamersaw in https://github.com/flyteorg/flyte/pull/4339
+* completing retries even if minSuccesses are achieved by @hamersaw in https://github.com/flyteorg/flyte/pull/4338
+* Add comment to auth scope by @wild-endeavor in https://github.com/flyteorg/flyte/pull/4341
+* Update tests by @eapolinario in https://github.com/flyteorg/flyte/pull/4381
+* Update order of cluster resources config to work with both uctl and flytectl by @neverett in https://github.com/flyteorg/flyte/pull/4373
+* Update tests in single-binary by @eapolinario in https://github.com/flyteorg/flyte/pull/4383
+* Passthrough unique node ID in task execution ID for generating log te… by @jeevb in https://github.com/flyteorg/flyte/pull/4380
+* Add Sections in the PR Template by @Future-Outlier in https://github.com/flyteorg/flyte/pull/4367
+* Update metadata in ArrayNode TaskExecutionEvents by @hamersaw in https://github.com/flyteorg/flyte/pull/4355
+* Fixes list formatting in flytepropeller arch docs by @thomasjpfan in https://github.com/flyteorg/flyte/pull/4345
+* Update boilerplate end2end tests by @hamersaw in https://github.com/flyteorg/flyte/pull/4393
+* Handle all ray job statuses by @EngHabu in https://github.com/flyteorg/flyte/pull/4389
+* Relocate sandbox config by @davidmirror-ops in https://github.com/flyteorg/flyte/pull/4385
+* Refactor task logs framework by @jeevb in https://github.com/flyteorg/flyte/pull/4396
+* Add support for displaying the Ray dashboard when a RayJob is active by @jeevb in https://github.com/flyteorg/flyte/pull/4397
+* Disable path filtering for monorepo components by @eapolinario in https://github.com/flyteorg/flyte/pull/4404
+* Silence NotFound when get task resource by @honnix in https://github.com/flyteorg/flyte/pull/4388
+* adding consoleUrl parameterization based on partition by @lauralindy in https://github.com/flyteorg/flyte/pull/4375
+* [Docs] Sensor Agent Doc by @Future-Outlier in https://github.com/flyteorg/flyte/pull/4195
+* [flytepropeller]  Add Tests in v1alpha.go including `array_test.go`, `branch_test.go`, `error_test.go`, and `iface_test.go` with 0.13% Coverage  Improvement by @Future-Outlier in https://github.com/flyteorg/flyte/pull/4234
+* Add more context for ray log template links by @jeevb in https://github.com/flyteorg/flyte/pull/4416
+* Add ClusterRole config for Ray by @davidmirror-ops in https://github.com/flyteorg/flyte/pull/4405
+* Fix and update scripts for generating grafana dashboards by @Tom-Newton in https://github.com/flyteorg/flyte/pull/4382
+* Add artifacts branch to publish to buf on push by @squiishyy in https://github.com/flyteorg/flyte/pull/4450
+* Add service monitor for flyte admin and propeller service by @vraiyaninv in https://github.com/flyteorg/flyte/pull/4427
+* Fix Kubeflow TF Operator `GetTaskPhase` Bug by @Future-Outlier in https://github.com/flyteorg/flyte/pull/4469
+* Instrument opentelemetry by @hamersaw in https://github.com/flyteorg/flyte/pull/4357
+* Delete the .github folder from each subdirectory by @pingsutw in https://github.com/flyteorg/flyte/pull/4480
+* Fix the loop variable scheduler issue by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/4468
+* Databricks Plugin Setup Doc Enhancement by @Future-Outlier in https://github.com/flyteorg/flyte/pull/4445
+* Put ticker back in place in propeller gc by @eapolinario in https://github.com/flyteorg/flyte/pull/4490
+* Store failed execution in flyteadmin by @iaroslav-ciupin in https://github.com/flyteorg/flyte/pull/4390
+* Moving from flyteadmin - Upgrade coreos/go-oidc to v3 to pickup claims parsing fixes by @eapolinario in https://github.com/flyteorg/flyte/pull/4139
+* Bump flyteorg/stow to 0.3.8 by @eapolinario in https://github.com/flyteorg/flyte/pull/4312
+* Remove 'needs' from generate_flyte_manifest by @eapolinario in https://github.com/flyteorg/flyte/pull/4495
+* Update Flyte components by @flyte-bot in https://github.com/flyteorg/flyte/pull/4302
+* Modify how flytecopilot version is parsed from values file by @eapolinario in https://github.com/flyteorg/flyte/pull/4496
+* Ignore component tags in goreleaser by @eapolinario in https://github.com/flyteorg/flyte/pull/4497
+* Fix indentation of `shell: task` by @eapolinario in https://github.com/flyteorg/flyte/pull/4498
+* Implemented simple echo plugin for testing by @hamersaw in https://github.com/flyteorg/flyte/pull/4489
+* Correctly handle resource overrides in KF plugins by @jeevb in https://github.com/flyteorg/flyte/pull/4467
+* Remove deprecated InjectDecoder by @EngHabu in https://github.com/flyteorg/flyte/pull/4507
+* Fix $HOME resolution and webhook namespace by @EngHabu in https://github.com/flyteorg/flyte/pull/4509
+* Add note on updating sandbox cluster configuration by @jeevb in https://github.com/flyteorg/flyte/pull/4510
+* Add New PR Template by @Future-Outlier in https://github.com/flyteorg/flyte/pull/4512
+* [Docs] Databricks Agent Doc by @Future-Outlier in https://github.com/flyteorg/flyte/pull/4008
+* Bump version of goreleaser gh action to v5 by @eapolinario in https://github.com/flyteorg/flyte/pull/4519
+* Kf operators use `GetReplicaFunc` (Error Handling) by @Future-Outlier in https://github.com/flyteorg/flyte/pull/4471
+
+## New Contributors
+* @HeetVekariya made their first contribution in https://github.com/flyteorg/flyte/pull/4252
+* @andresgomezfrr made their first contribution in https://github.com/flyteorg/flyte/pull/4342
+* @brndnblck made their first contribution in https://github.com/flyteorg/flyte/pull/4352
+* @guyarad made their first contribution in https://github.com/flyteorg/flyte/pull/4337
+* @neverett made their first contribution in https://github.com/flyteorg/flyte/pull/4373
+* @thomasjpfan made their first contribution in https://github.com/flyteorg/flyte/pull/4345
+* @lauralindy made their first contribution in https://github.com/flyteorg/flyte/pull/4375
+* @Tom-Newton made their first contribution in https://github.com/flyteorg/flyte/pull/4382
+* @vraiyaninv made their first contribution in https://github.com/flyteorg/flyte/pull/4427
diff --git a/CHANGELOG/CHANGELOG-v1.3.0-b5.md b/CHANGELOG/CHANGELOG-v1.3.0-b5.md
@@ -99,7 +99,7 @@ flytectl demo start --image ghcr.io/flyteorg/flyte-sandbox-bundled:sha-e240038be
    ```
 
 ### Databricks Code
-You'll need to upload an [entrypoint](https://gist.github.com/pingsutw/482e7f0134414dac437500344bac5134) file to your dbfs (or S3). This is the referenced gist from the primary [Databricks plugin documentation](https://github.com/flyteorg/flyte/blob/master/rsts/deployment/plugin_setup/webapi/databricks.rst) as well, which currently only covers the `flyte-core` Helm chart installation.
+You'll need to upload an [entrypoint](https://gist.github.com/pingsutw/482e7f0134414dac437500344bac5134) file to your dbfs (or S3). This is the referenced gist from the primary [Databricks plugin documentation](https://github.com/flyteorg/flyte/blob/master/docs/deployment/plugin_setup/webapi/databricks.rst) as well, which currently only covers the `flyte-core` Helm chart installation.
 
 
 ### User Code