Merge remote-tracking branch 'origin' into monorepo-move/flyteadmin/u…

…pgrade-coreos/go-oidc

Signed-off-by: Eduardo Apolinario <[email protected]>

.github/workflows/checks.yml

@@ -1,12 +1,15 @@
 name: Components Checks
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 on:
   pull_request:
     paths:
       - 'datacatalog/**'
       - 'flyteadmin/**'
       - 'flytecopilot/**'
-      - 'flyteidl/**'
       - 'flyteplugins/**'
       - 'flytepropeller/**'
       - 'flytestdlib/**'
@@ -65,8 +68,6 @@ jobs:
           - datacatalog
           - flyteadmin
           - flytecopilot
-          # TODO(monorepo): Enable flyteidl unit tests
-          # - flyteidl
           - flyteplugins
           - flytepropeller
           - flytestdlib
@@ -133,47 +134,11 @@ jobs:
       component: ${{ matrix.component }}
       go-version: ${{ needs.unpack-envvars.outputs.go-version }}
 
-  bump-tags:
-    name: Bump git tags
-    # TODO(monorepo): skip this if author is flyte-bot?
-    if: ${{ github.event_name != 'pull_request' }}
-    needs:
-      - integration
-      - lint
-      - unit-tests
-      - generate
-    uses: ./.github/workflows/bump-tags.yml
-    secrets:
-      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
-
-  # TODO(monorepo): we are not going to release any binary
-  # goreleaser:
-  #   name: Goreleaser
-  #   needs: [ bump_version ] # Only to ensure it can successfully build
-  #   uses: flyteorg/flytetools/.github/workflows/goreleaser.yml@master
-  #   with:
-  #     go-version: "1.19"
-  #   secrets:
-  #     FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
-
-  push_docker_image:
-    name: Build & Push Image
-    needs: [ bump-tags ]
-    strategy:
-      fail-fast: false
-      matrix:
-        component:
-          - datacatalog
-          - flyteadmin
-          - flytecopilot
-          - flytepropeller
-          - flytescheduler
-    uses: ./.github/workflows/publish.yml
+  build_docker_images:
+    name: Build Images
+    uses: ./.github/workflows/publish-images.yml
     with:
-      version: ${{ needs.bump-tags.outputs.next_version }}
-      component: ${{ matrix.component }}
-      dockerfile: Dockerfile.${{ matrix.component }}
-      push: true
+      push: false
     secrets:
       FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
       FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }}

.github/workflows/codespell.yml

@@ -1,6 +1,10 @@
 ---
 name: Codespell
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 on:
   push:
     branches: [master]
@@ -20,3 +24,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Codespell
         uses: codespell-project/actions-codespell@v2
+        with:
+          skip: "*.pb"

.github/workflows/create_release.yml

@@ -7,29 +7,57 @@ on:
         required: true
 
 jobs:
-  bump-version:
-    name: bump-version
+  generate-tags:
+    name: Generate git tags
     runs-on: ubuntu-latest
-    outputs:
-      version: ${{ steps.bump-version.outputs.tag }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
-          fetch-depth: "0"
+          fetch-depth: '0'
+      - uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.FLYTE_BOT_PAT }}
+          script: |
+            github.rest.git.createRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: `refs/tags/${{ github.event.inputs.version }}`,
+              sha: context.sha
+            })
+            const components = [
+              "datacatalog",
+              "flyteadmin",
+              "flytecopilot",
+              "flyteidl",
+              "flyteplugins",
+              "flytepropeller",
+              "flytestdlib",
+            ];
+            for (const c of components) {
+              github.rest.git.createRef({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                ref: `refs/tags/${c}/${{ github.event.inputs.version }}`,
+                sha: context.sha
+              })
+            }
 
-      - name: Bump version and push tag
-        id: bump-version
-        uses: anothrNick/[email protected]
-        env:
-          GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }}
-          WITH_V: true
-          CUSTOM_TAG: ${{ github.event.inputs.version }}
-          RELEASE_BRANCHES: master
+  build-docker-images:
+    needs:
+      - generate-tags
+    uses: ./.github/workflows/publish-images.yml
+    with:
+      version: ${{ github.event.inputs.version }}
+      push: true
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+      FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }}
 
   publish-flyte-binary-image:
     name: Publish flyte binary image for the release version
     runs-on: ubuntu-latest
-    needs: [bump-version]
+    needs:
+      - generate-tags
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -45,24 +73,25 @@ jobs:
 
       - name: Tag image to release version
         run: |
-          for release in latest ${{ needs.bump-version.outputs.version }}; do
+          for release in latest ${{ github.event.inputs.version }}; do
             docker buildx imagetools create --tag "ghcr.io/${{ github.repository_owner }}/flyte-binary-release:${release}" "ghcr.io/${{ github.repository_owner }}/flyte-binary:sha-${{ github.sha }}"
           done
 
   publish-flyte-component-image:
     name: Publish flyte component image for the release version
     runs-on: ubuntu-latest
-    needs: [bump-version]
+    needs:
+      - build-docker-images
     strategy:
       matrix:
         component:
           [
+              datacatalog,
+              flyteadmin,
               flyteconsole,
+              flytecopilot,
               flytepropeller,
-              flyteadmin,
-              datacatalog,
               flytescheduler,
-              flytecopilot,
           ]
     steps:
       - name: Checkout
@@ -76,11 +105,7 @@ jobs:
       - name: Get Latest Version of component
         id: set_version
         run: |
-          if [ ${{ matrix.component }} = "flytecopilot" ]; then
-             echo ::set-output name=version::$(yq eval '.configmap.copilot.plugins.k8s.co-pilot.image' charts/flyte-core/values.yaml | cut -d ":" -f 2 )
-          else
-             echo ::set-output name=version::$(yq eval '.${{ matrix.component }}.image.tag' charts/flyte-core/values.yaml)
-          fi
+          echo ::set-output name=version::$(yq eval '.${{ matrix.component }}.image.tag' charts/flyte-core/values.yaml)
         shell: bash
 
       - name: Login to GitHub Container Registry
@@ -92,14 +117,15 @@ jobs:
 
       - name: Tag Image to release version
         run: |
-          for release in latest ${{ needs.bump-version.outputs.version }}; do
+          for release in latest ${{ github.event.inputs.version }}; do
             docker buildx imagetools create --tag "ghcr.io/${{ github.repository_owner }}/${{ matrix.component }}-release:${release}" "ghcr.io/${{ github.repository_owner }}/${{ matrix.component }}:${{ steps.set_version.outputs.version }}"
           done
 
   helm-release:
     name: Flyte helm release
     runs-on: ubuntu-latest
-    needs: [bump-version]
+    needs:
+      - build-docker-images
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -118,7 +144,7 @@ jobs:
           git config user.email "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>"
       - name: Prepare Flyte Helm Release
         env:
-          VERSION: ${{ needs.bump-version.outputs.version }}
+          VERSION: ${{ github.event.inputs.version }}
           REPOSITORY: "https://flyteorg.github.io/flyte"
         run: |
           make prepare_artifacts
@@ -131,7 +157,8 @@ jobs:
   manifest-release:
     name: Flyte manifest release
     runs-on: ubuntu-latest
-    needs: [bump-version]
+    needs:
+      - build-docker-images
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -140,7 +167,7 @@ jobs:
 
       - name: Prepare Flyte Release
         env:
-          VERSION: ${{ needs.bump-version.outputs.version }}
+          VERSION: ${{ github.event.inputs.version }}
         run: |
           make prepare_artifacts
           git stash

.github/workflows/flyteidl-buf-publish.yml

@@ -0,0 +1,18 @@
+name: Publish flyteidl Buf Package
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'flyteidl/**'
+jobs:
+  buf:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: bufbuild/buf-setup-action@v1
+      - uses: bufbuild/buf-push-action@v1
+        with:
+          buf_token: ${{ secrets.BUF_TOKEN }}
+          input: 'flyteidl/protos'

.github/workflows/flyteidl-checks.yml

@@ -0,0 +1,56 @@
+name: Flyteidl Verification Tests
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    paths:
+      - 'flyteidl/**'
+  push:
+    branches:
+      - master
+    paths:
+      - 'flyteidl/**'
+env:
+  GO_VERSION: "1.19"
+jobs:
+  unpack-envvars:
+    runs-on: ubuntu-latest
+    outputs:
+      go-version: ${{ steps.step.outputs.go-version }}
+    steps:
+      - id: step
+        run: |
+          echo "go-version=${{ env.GO_VERSION }}" >> $GITHUB_OUTPUT
+  lint:
+    name: Lint
+    needs:
+      - unpack-envvars
+    strategy:
+      fail-fast: false
+    uses: ./.github/workflows/lint.yml
+    with:
+      component: flyteidl
+      go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+  unit-tests:
+    name: Unit Tests
+    needs:
+      - unpack-envvars
+    uses: ./.github/workflows/unit-tests.yml
+    with:
+      component: flyteidl
+      go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+  generate:
+    name: Check Go Generate
+    needs:
+      - unpack-envvars
+    strategy:
+      fail-fast: false
+    uses: ./.github/workflows/go_generate.yml
+    with:
+      component: flyteidl
+      go-version: ${{ needs.unpack-envvars.outputs.go-version }}