Merge branch 'master' into flyte-core-updates

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,38 +1,51 @@
-<!--
-Thank you for sending the PR! 
-Please fill the applicable details below
-Happy contributing!
--->
-
 ## Tracking issue
+_https://github.com/flyteorg/flyte/issues/<number>_
 
 <!-- If your PR fixes an open issue, use `Closes #999` to link your PR with the issue. #999 stands for the issue number you are fixing -->
 
 <!-- Remove this section if not applicable -->
 
 <!-- Example: Closes #31 -->
 
-## Describe your changes
+## Why are the changes needed?
+
+<!--
+Please clarify why the changes are needed. For instance,
+1. If you propose a new API, clarify the use case for a new API.
+2. If you fix a bug, you can clarify why it is a bug.
+-->
+
+## What changes were proposed in this pull request?
+
+<!--
+Please clarify what changes you are proposing. The purpose of this section is to outline the changes and how this PR fixes the issue.
+If possible, please consider writing useful notes for better and faster reviews in your PR. See the examples below.
+1. If you refactor some codes with changing classes, showing the class hierarchy will help reviewers.
+2. If there is design documentation, please add the link.
+-->
 
-<!-- List all the proposed changes in your PR -->
+## How was this patch tested?
 
-<!-- Mark all the applicable boxes. To mark the box as done follow the following conventions -->
 <!--
-[x] - Correct; marked as done
-[X] - Correct; marked as done
-[ ] - Not correct; marked as **not** done
+If tests were added, say they were added here. Please make sure to add some test cases that check the changes thoroughly including negative and positive cases if possible.
+If it was tested in a way different from regular unit tests, please clarify how you tested step by step, ideally copy and paste-able, so that other reviewers can test and check, and descendants can verify in the future.
+If tests were not added, please describe why they were not added and/or why it was difficult to add.
 -->
 
+### Setup process
+
+### Screenshots
+
 ## Check all the applicable boxes <!-- Follow the above conventions to check the box -->
 
 - [ ] I updated the documentation accordingly.
 - [ ] All new and existing tests passed.
 - [ ] All commits are signed-off.
 
-## Screenshots
+## Related PRs
 
-<!-- Add all the screenshots which support your changes -->
+<!-- Add related pull requests for reviewers to check -->
 
-## Note to reviewers
+## Docs link
 
-<!-- Add notes to reviewers if applicable -->
+<!-- Add documentation link built by CI jobs here, and specify the changed place -->

.github/workflows/checks.yml

@@ -6,24 +6,9 @@ concurrency:
 
 on:
   pull_request:
-    paths:
-      - 'datacatalog/**'
-      - 'flyteadmin/**'
-      - 'flytecopilot/**'
-      - 'flyteplugins/**'
-      - 'flytepropeller/**'
-      - 'flytestdlib/**'
   push:
     branches:
       - master
-    paths:
-      - 'datacatalog/**'
-      - 'flyteadmin/**'
-      - 'flytecopilot/**'
-      - 'flyteidl/**'
-      - 'flyteplugins/**'
-      - 'flytepropeller/**'
-      - 'flytestdlib/**'
 env:
   GO_VERSION: "1.19"
   PRIORITIES: "P0"
@@ -133,48 +118,14 @@ jobs:
     with:
       component: ${{ matrix.component }}
       go-version: ${{ needs.unpack-envvars.outputs.go-version }}
-
-  bump-tags:
-    name: Bump git tags
-    # TODO(monorepo): skip this if author is flyte-bot?
-    if: ${{ github.event_name != 'pull_request' }}
-    needs:
-      - integration
-      - lint
-      - unit-tests
-      - generate
-    uses: ./.github/workflows/bump-tags.yml
     secrets:
       FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
 
-  # TODO(monorepo): we are not going to release any binary
-  # goreleaser:
-  #   name: Goreleaser
-  #   needs: [ bump_version ] # Only to ensure it can successfully build
-  #   uses: flyteorg/flytetools/.github/workflows/goreleaser.yml@master
-  #   with:
-  #     go-version: "1.19"
-  #   secrets:
-  #     FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
-
-  push_docker_image:
-    name: Build & Push Image
-    needs: [ bump-tags ]
-    strategy:
-      fail-fast: false
-      matrix:
-        component:
-          - datacatalog
-          - flyteadmin
-          - flytecopilot
-          - flytepropeller
-          - flytescheduler
-    uses: ./.github/workflows/publish.yml
+  build_docker_images:
+    name: Build Images
+    uses: ./.github/workflows/publish-images.yml
     with:
-      version: ${{ needs.bump-tags.outputs.next_version }}
-      component: ${{ matrix.component }}
-      dockerfile: Dockerfile.${{ matrix.component }}
-      push: true
+      push: false
     secrets:
       FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
       FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }}

.github/workflows/codespell.yml

@@ -25,4 +25,4 @@ jobs:
       - name: Codespell
         uses: codespell-project/actions-codespell@v2
         with:
-          skip: "*.pb"
+          skip: "*.pb,monodocs-environment.lock.yaml"

.github/workflows/create_release.yml

@@ -7,29 +7,57 @@ on:
         required: true
 
 jobs:
-  bump-version:
-    name: bump-version
+  generate-tags:
+    name: Generate git tags
     runs-on: ubuntu-latest
-    outputs:
-      version: ${{ steps.bump-version.outputs.tag }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
-          fetch-depth: "0"
+          fetch-depth: '0'
+      - uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.FLYTE_BOT_PAT }}
+          script: |
+            github.rest.git.createRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: `refs/tags/${{ github.event.inputs.version }}`,
+              sha: context.sha
+            })
+            const components = [
+              "datacatalog",
+              "flyteadmin",
+              "flytecopilot",
+              "flyteidl",
+              "flyteplugins",
+              "flytepropeller",
+              "flytestdlib",
+            ];
+            for (const c of components) {
+              github.rest.git.createRef({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                ref: `refs/tags/${c}/${{ github.event.inputs.version }}`,
+                sha: context.sha
+              })
+            }
 
-      - name: Bump version and push tag
-        id: bump-version
-        uses: anothrNick/[email protected]
-        env:
-          GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }}
-          WITH_V: true
-          CUSTOM_TAG: ${{ github.event.inputs.version }}
-          RELEASE_BRANCHES: master
+  build-docker-images:
+    needs:
+      - generate-tags
+    uses: ./.github/workflows/publish-images.yml
+    with:
+      version: ${{ github.event.inputs.version }}
+      push: true
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+      FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }}
 
   publish-flyte-binary-image:
     name: Publish flyte binary image for the release version
     runs-on: ubuntu-latest
-    needs: [bump-version]
+    needs:
+      - generate-tags
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -45,24 +73,25 @@ jobs:
 
       - name: Tag image to release version
         run: |
-          for release in latest ${{ needs.bump-version.outputs.version }}; do
+          for release in latest ${{ github.event.inputs.version }}; do
             docker buildx imagetools create --tag "ghcr.io/${{ github.repository_owner }}/flyte-binary-release:${release}" "ghcr.io/${{ github.repository_owner }}/flyte-binary:sha-${{ github.sha }}"
           done
 
   publish-flyte-component-image:
     name: Publish flyte component image for the release version
     runs-on: ubuntu-latest
-    needs: [bump-version]
+    needs:
+      - build-docker-images
     strategy:
       matrix:
         component:
           [
+              datacatalog,
+              flyteadmin,
               flyteconsole,
+              flytecopilot,
               flytepropeller,
-              flyteadmin,
-              datacatalog,
               flytescheduler,
-              flytecopilot,
           ]
     steps:
       - name: Checkout
@@ -92,14 +121,15 @@ jobs:
 
       - name: Tag Image to release version
         run: |
-          for release in latest ${{ needs.bump-version.outputs.version }}; do
+          for release in latest ${{ github.event.inputs.version }}; do
             docker buildx imagetools create --tag "ghcr.io/${{ github.repository_owner }}/${{ matrix.component }}-release:${release}" "ghcr.io/${{ github.repository_owner }}/${{ matrix.component }}:${{ steps.set_version.outputs.version }}"
           done
 
   helm-release:
     name: Flyte helm release
     runs-on: ubuntu-latest
-    needs: [bump-version]
+    needs:
+      - build-docker-images
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -118,7 +148,7 @@ jobs:
           git config user.email "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>"
       - name: Prepare Flyte Helm Release
         env:
-          VERSION: ${{ needs.bump-version.outputs.version }}
+          VERSION: ${{ github.event.inputs.version }}
           REPOSITORY: "https://flyteorg.github.io/flyte"
         run: |
           make prepare_artifacts
@@ -131,7 +161,8 @@ jobs:
   manifest-release:
     name: Flyte manifest release
     runs-on: ubuntu-latest
-    needs: [bump-version]
+    needs:
+      - build-docker-images
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -140,13 +171,13 @@ jobs:
 
       - name: Prepare Flyte Release
         env:
-          VERSION: ${{ needs.bump-version.outputs.version }}
+          VERSION: ${{ github.event.inputs.version }}
         run: |
           make prepare_artifacts
           git stash
 
       - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v2
+        uses: goreleaser/goreleaser-action@v5
         with:
           version: latest
           args: release --rm-dist --release-notes=CHANGELOG/CHANGELOG-${{ github.event.inputs.version }}.md

.github/workflows/end2end.yml

@@ -79,26 +79,16 @@ jobs:
       - name: Register P0 tests
         if: ${{ inputs.priorities == 'P0' }}
         run: |
-          for f in \
-                  basics/basics/hello_world.py \
-                  basics/basics/workflow.py \
-                  basics/basics/named_outputs.py \
-                  advanced_composition/advanced_composition/chain_entities.py \
-                  advanced_composition/advanced_composition/dynamics.py \
-                  advanced_composition/advanced_composition/map_task.py \
-                  advanced_composition/advanced_composition/subworkflows.py \
-                  data_types_and_io/data_types_and_io/custom_objects.py \
-                  data_types_and_io/data_types_and_io/schema.py \
-                  data_types_and_io/data_types_and_io/typed_schema.py ;
+          while read -r line;
           do
               pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
                   register \
                   --project flytesnacks \
                   --domain development \
                   --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \
                   --version ${{ env.FLYTESNACKS_VERSION }} \
-                  flytesnacks/examples/$f;
-          done
+                  flytesnacks/$line;
+          done < flytesnacks/flyte_tests.txt
       - name: Register all flytesnacks examples
         if: ${{ inputs.priorities != 'P0' }}
         uses: unionai/[email protected]