Merge branch 'master' into add-ray-autoscaler-config

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,38 +1,51 @@
-<!--
-Thank you for sending the PR! 
-Please fill the applicable details below
-Happy contributing!
--->
-
 ## Tracking issue
+_https://github.com/flyteorg/flyte/issues/<number>_
 
 <!-- If your PR fixes an open issue, use `Closes #999` to link your PR with the issue. #999 stands for the issue number you are fixing -->
 
 <!-- Remove this section if not applicable -->
 
 <!-- Example: Closes #31 -->
 
-## Describe your changes
+## Why are the changes needed?
+
+<!--
+Please clarify why the changes are needed. For instance,
+1. If you propose a new API, clarify the use case for a new API.
+2. If you fix a bug, you can clarify why it is a bug.
+-->
+
+## What changes were proposed in this pull request?
+
+<!--
+Please clarify what changes you are proposing. The purpose of this section is to outline the changes and how this PR fixes the issue.
+If possible, please consider writing useful notes for better and faster reviews in your PR. See the examples below.
+1. If you refactor some codes with changing classes, showing the class hierarchy will help reviewers.
+2. If there is design documentation, please add the link.
+-->
 
-<!-- List all the proposed changes in your PR -->
+## How was this patch tested?
 
-<!-- Mark all the applicable boxes. To mark the box as done follow the following conventions -->
 <!--
-[x] - Correct; marked as done
-[X] - Correct; marked as done
-[ ] - Not correct; marked as **not** done
+If tests were added, say they were added here. Please make sure to add some test cases that check the changes thoroughly including negative and positive cases if possible.
+If it was tested in a way different from regular unit tests, please clarify how you tested step by step, ideally copy and paste-able, so that other reviewers can test and check, and descendants can verify in the future.
+If tests were not added, please describe why they were not added and/or why it was difficult to add.
 -->
 
+### Setup process
+
+### Screenshots
+
 ## Check all the applicable boxes <!-- Follow the above conventions to check the box -->
 
 - [ ] I updated the documentation accordingly.
 - [ ] All new and existing tests passed.
 - [ ] All commits are signed-off.
 
-## Screenshots
+## Related PRs
 
-<!-- Add all the screenshots which support your changes -->
+<!-- Add related pull requests for reviewers to check -->
 
-## Note to reviewers
+## Docs link
 
-<!-- Add notes to reviewers if applicable -->
+<!-- Add documentation link built by CI jobs here, and specify the changed place -->

.github/workflows/checks.yml

@@ -6,26 +6,11 @@ concurrency:
 
 on:
   pull_request:
-    paths:
-      - 'datacatalog/**'
-      - 'flyteadmin/**'
-      - 'flytecopilot/**'
-      - 'flyteplugins/**'
-      - 'flytepropeller/**'
-      - 'flytestdlib/**'
   push:
     branches:
       - master
-    paths:
-      - 'datacatalog/**'
-      - 'flyteadmin/**'
-      - 'flytecopilot/**'
-      - 'flyteidl/**'
-      - 'flyteplugins/**'
-      - 'flytepropeller/**'
-      - 'flytestdlib/**'
 env:
-  GO_VERSION: "1.19"
+  GO_VERSION: "1.21"
   PRIORITIES: "P0"
 jobs:
   unpack-envvars:
@@ -133,6 +118,8 @@ jobs:
     with:
       component: ${{ matrix.component }}
       go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
 
   build_docker_images:
     name: Build Images

.github/workflows/codespell.yml

@@ -25,4 +25,4 @@ jobs:
       - name: Codespell
         uses: codespell-project/actions-codespell@v2
         with:
-          skip: "*.pb"
+          skip: "*.pb,monodocs-environment.lock.yaml"

.github/workflows/create_release.yml

@@ -105,7 +105,11 @@ jobs:
       - name: Get Latest Version of component
         id: set_version
         run: |
-          echo ::set-output name=version::$(yq eval '.${{ matrix.component }}.image.tag' charts/flyte-core/values.yaml)
+          if [ ${{ matrix.component }} = "flytecopilot" ]; then
+             echo ::set-output name=version::$(yq eval '.configmap.copilot.plugins.k8s.co-pilot.image' charts/flyte-core/values.yaml | cut -d ":" -f 2 )
+          else
+             echo ::set-output name=version::$(yq eval '.${{ matrix.component }}.image.tag' charts/flyte-core/values.yaml)
+          fi
         shell: bash
 
       - name: Login to GitHub Container Registry
@@ -173,7 +177,7 @@ jobs:
           git stash
 
       - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v2
+        uses: goreleaser/goreleaser-action@v5
         with:
           version: latest
           args: release --rm-dist --release-notes=CHANGELOG/CHANGELOG-${{ github.event.inputs.version }}.md

.github/workflows/end2end.yml

@@ -79,26 +79,16 @@ jobs:
       - name: Register P0 tests
         if: ${{ inputs.priorities == 'P0' }}
         run: |
-          for f in \
-                  basics/basics/hello_world.py \
-                  basics/basics/workflow.py \
-                  basics/basics/named_outputs.py \
-                  advanced_composition/advanced_composition/chain_entities.py \
-                  advanced_composition/advanced_composition/dynamics.py \
-                  advanced_composition/advanced_composition/map_task.py \
-                  advanced_composition/advanced_composition/subworkflows.py \
-                  data_types_and_io/data_types_and_io/custom_objects.py \
-                  data_types_and_io/data_types_and_io/schema.py \
-                  data_types_and_io/data_types_and_io/typed_schema.py ;
+          while read -r line;
           do
               pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
                   register \
                   --project flytesnacks \
                   --domain development \
                   --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \
                   --version ${{ env.FLYTESNACKS_VERSION }} \
-                  flytesnacks/examples/$f;
-          done
+                  flytesnacks/$line;
+          done < flytesnacks/flyte_tests.txt
       - name: Register all flytesnacks examples
         if: ${{ inputs.priorities != 'P0' }}
         uses: unionai/[email protected]

.github/workflows/flyteidl-buf-publish.yml

@@ -3,6 +3,7 @@ name: Publish flyteidl Buf Package
 on:
   push:
     branches:
+      - artifacts
       - master
     paths:
       - 'flyteidl/**'

.github/workflows/flyteidl-checks.yml

@@ -6,15 +6,11 @@ concurrency:
 
 on:
   pull_request:
-    paths:
-      - 'flyteidl/**'
   push:
     branches:
       - master
-    paths:
-      - 'flyteidl/**'
 env:
-  GO_VERSION: "1.19"
+  GO_VERSION: "1.21"
 jobs:
   unpack-envvars:
     runs-on: ubuntu-latest

.github/workflows/generate_flyte_manifest.yml

@@ -11,15 +11,13 @@ jobs:
   update-flyte-releases:
     name: Update Flyte components
     runs-on: ubuntu-latest
-    needs:
-      - generate-tags
     steps:
       - uses: actions/checkout@v2
         with:
           fetch-depth: "0"
       - uses: actions/setup-go@v2
         with:
-          go-version: "1.19"
+          go-version: "1.21"
       - name: Update references
         env:
           VERSION: ${{ github.event.inputs.next-version }}

.github/workflows/go_generate.yml

@@ -9,6 +9,9 @@ on:
       go-version:
         required: true
         type: string
+    secrets:
+      FLYTE_BOT_PAT:
+        required: true
 jobs:
   generate:
     runs-on: ubuntu-latest
@@ -19,7 +22,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: arduino/setup-protoc@v1
+        with:
+          repo-token: ${{ secrets.FLYTE_BOT_PAT }}
       - uses: bufbuild/buf-setup-action@v1
+        with:
+          github_token: ${{ secrets.FLYTE_BOT_PAT }}
       - uses: actions/setup-go@v3
         with:
           go-version: ${{ inputs.go-version }}

.github/workflows/single-binary.yml

@@ -6,14 +6,6 @@ concurrency:
 
 on:
   pull_request:
-    paths:
-      - .github/workflows/single-binary.yml
-      - charts/flyte-binary/**
-      - charts/flyte-sandbox/**
-      - cmd/**
-      - docker/sandbox-bundled/**
-      - Dockerfile
-      - go.*
   push:
     branches:
       - master
@@ -27,7 +19,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.19'
+          go-version: "1.21"
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
@@ -129,9 +121,9 @@ jobs:
   sandbox-bundled-functional-tests:
     runs-on: ubuntu-latest
     env:
-      FLYTESNACKS_PRIORITIES: 'P0'
-      FLYTESNACKS_VERSION: ''
-    timeout-minutes: 20
+      FLYTESNACKS_PRIORITIES: "P0"
+      FLYTESNACKS_VERSION: ""
+    timeout-minutes: 60
     needs: [build-and-push-single-binary-image]
     steps:
       - name: Set latest Flytesnacks release
@@ -164,7 +156,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.11'
+          python-version: "3.11"
       - uses: unionai/[email protected]
       - name: Setup sandbox
         run: |
@@ -193,26 +185,16 @@ jobs:
           # ref: ${{ env.FLYTESNACKS_VERSION }}
       - name: Register specific tests
         run: |
-          for f in \
-                  basics/basics/hello_world.py \
-                  basics/basics/workflow.py \
-                  basics/basics/named_outputs.py \
-                  advanced_composition/advanced_composition/chain_entities.py \
-                  advanced_composition/advanced_composition/dynamics.py \
-                  advanced_composition/advanced_composition/map_task.py \
-                  advanced_composition/advanced_composition/subworkflows.py \
-                  data_types_and_io/data_types_and_io/custom_objects.py \
-                  data_types_and_io/data_types_and_io/schema.py \
-                  data_types_and_io/data_types_and_io/typed_schema.py ;
+          while read -r line;
           do
               pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
                   register \
                   --project flytesnacks \
                   --domain development \
                   --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \
                   --version ${{ env.FLYTESNACKS_VERSION }} \
-                  flytesnacks/examples/$f;
-          done
+                  flytesnacks/$line;
+          done < flytesnacks/flyte_tests.txt
       - name: End2End
         run: |
           make end2end_execute

.github/workflows/tests.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: "1.19"
+          go-version: "1.21"
       - name: Compile
         run: make compile
       - name: Run tests
@@ -28,25 +28,38 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Fetch the code
-        uses: actions/checkout@v2
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+        uses: actions/checkout@v4
+      - uses: conda-incubator/setup-miniconda@v3
         with:
-          python-version: "3.8"
-      - name: Install dependencies
+          auto-update-conda: true
+          python-version: 3.9
+      - shell: bash -el {0}
+        run: |
+          conda install -c conda-forge conda-lock
+          conda-lock install -n monodocs-env monodocs-environment.lock.yaml
+      - shell: bash -el {0}
         run: |
-          python -m pip install --upgrade pip
-          if [ -f doc-requirements.txt ]; then pip install -r doc-requirements.txt; fi
+          conda activate monodocs-env
+          conda info
+          conda list
+          conda config --show-sources
+          conda config --show
+          printenv | sort
+      - name: Setup Graphviz
+        uses: ts-graphviz/setup-graphviz@v1
       - name: Build the documentation
-        run: make docs
+        shell: bash -el {0}
+        run: |
+          conda activate monodocs-env
+          make docs
 
   generate_kustomize:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: "1.19"
+          go-version: "1.21"
       - name: Kustomize and diff
         run: DELTA_CHECK=true make kustomize
 
@@ -56,6 +69,6 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: "1.19"
+          go-version: "1.21"
       - name: Helm and diff
         run: DELTA_CHECK=true make helm

.gitignore

@@ -10,10 +10,6 @@ _repos/
 _rsts/
 rsts_tmp/
 .doctrees/
-docs/_sources/
-docs/flytekit/flytekit.interfaces.html
-docs/searchindex.js
-docs/
 !flyteidl/protos/docs
 release/
 __pycache__/
@@ -33,5 +29,14 @@ dist
 *.db
 vendor/
 /docker/sandbox-bundled/images/tar
-rsts/_tags/
-**/bin/
+**/bin/
+docs/_tags/
+docs/flytectl
+docs/protos
+docs/flytekit
+docs/flytesnacks
+docs/examples
+docs/_src
+docs/_projects
+docs/api
+docs/tests