Merge remote-tracking branch 'origin' into rliu.redirect-url

Signed-off-by: Eduardo Apolinario <[email protected]>

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,11 +1,13 @@
 ## Tracking issue
-_https://github.com/flyteorg/flyte/issues/<number>_
-
-<!-- If your PR fixes an open issue, use `Closes #999` to link your PR with the issue. #999 stands for the issue number you are fixing -->
+<!--
+If your PR fixes an open issue, use `Closes #999` to link your PR with the issue.
+Example: Closes #999
 
-<!-- Remove this section if not applicable -->
+If your PR is related to an issue or PR, use `Related to #999` to link your PR.
+Example: Related to #999
 
-<!-- Example: Closes #31 -->
+Remove this section if not applicable
+-->
 
 ## Why are the changes needed?
 

.github/codespell-ignored-words

@@ -0,0 +1,11 @@
+ThirdParty
+bootup
+crate
+fo
+lightyear
+nd
+notin
+querys
+ser
+te
+updAt

.github/workflows/checks.yml

@@ -34,6 +34,7 @@ jobs:
           - flyteadmin
           # TODO(monorepo): Enable lint flytecopilot
           # - flytecopilot
+          - flytectl
           - flyteidl
           - flyteplugins
           - flytepropeller
@@ -53,6 +54,7 @@ jobs:
           - datacatalog
           - flyteadmin
           - flytecopilot
+          - flytectl
           - flyteplugins
           - flytepropeller
           - flytestdlib
@@ -61,35 +63,20 @@ jobs:
       component: ${{ matrix.component }}
       go-version: ${{ needs.unpack-envvars.outputs.go-version }}
     secrets:
-      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
-  docker-build:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  flyteadmin-int-tests-image-build:
     strategy:
       fail-fast: false
-      matrix:
-        component:
-          - datacatalog
-          - flyteadmin
-          - flytecopilot
-          - flytepropeller
-    name: Docker Build Images
+    name: flyteadmin integration tests image 
     uses: ./.github/workflows/component_docker_build.yml
     with:
-       component: ${{ matrix.component }}
+      component: flyteadmin
 
-  # TODO(monorepo): these tests are broken. They never test an actual change.
-  # endtoend:
-  #   name: End2End Test
-  #   needs: [ docker-build ]
-  #   uses: ./.github/workflows/end2end.yml
-  #   with:
-  #     # Reusing the output of the matrix is ok as they are essentially writing the same value (i.e. the directory artifacts are written + run id)
-  #     cache_key: ${{ needs.docker-build.outputs.cache_key }}
-  #     priorities: "P0"
-
-  integration:
+  flyteadmin-integration-tests:
     name: Integration Test
     needs:
-      - docker-build
+      - flyteadmin-int-tests-image-build
       - unpack-envvars
     strategy:
       fail-fast: false
@@ -99,7 +86,7 @@ jobs:
     uses: ./.github/workflows/integration.yml
     with:
       component: ${{ matrix.component }}
-      cache_key: ${{ needs.docker-build.outputs.cache_key }}
+      cache_key: ${{ needs.flyteadmin-int-tests-image-build.outputs.cache_key }}
       go-version: ${{ needs.unpack-envvars.outputs.go-version }}
 
   generate:
@@ -113,6 +100,7 @@ jobs:
           - datacatalog
           - flyteadmin
           - flytecopilot
+          - flytectl
           - flytepropeller
     uses: ./.github/workflows/go_generate.yml
     with:
@@ -129,3 +117,59 @@ jobs:
     secrets:
       FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
       FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }}
+
+  dry_run_goreleaser:
+    name: Dry Run Goreleaser
+    runs-on: ubuntu-latest
+    needs:
+      - unpack-envvars
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: "0"
+      - uses: actions/setup-go@v4
+        with:
+          go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+      - name: Run GoReleaser dry run
+        uses: goreleaser/goreleaser-action@v5
+        with:
+          distribution: goreleaser-pro
+          version: latest
+          args: --snapshot  --skip-publish --rm-dist -f flytectl/.goreleaser.yml
+        env:
+          GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
+
+  sandbox:
+    name: Test Getting started
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: flytectl
+    needs:
+      - unpack-envvars
+    steps:
+      - uses: insightsengineering/disk-space-reclaimer@v1
+      - name: Checkout
+        uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+      - name: Build Flytectl binary
+        run: make compile
+      - name: Create a sandbox cluster
+        run: |
+          bin/flytectl demo start
+      - name: Setup flytectl config
+        run: bin/flytectl config init
+      - name: Register cookbook
+        run: bin/flytectl register examples -d development  -p flytesnacks
+      - name: Teardown Sandbox cluster
+        run: bin/flytectl demo teardown

.github/workflows/codespell.yml

@@ -21,8 +21,9 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Codespell
         uses: codespell-project/actions-codespell@v2
         with:
-          skip: "*.pb,monodocs-environment.lock.yaml"
+          skip: "*.pb,monodocs-environment.lock.yaml,.git,*.pdf,*.svg,go.sum,go.mod,*requirements.txt,gen"
+          ignore_words_file: .github/codespell-ignored-words

.github/workflows/component_docker_build.yml

@@ -17,7 +17,7 @@ jobs:
       cache_key: ${{ steps.cache_key.outputs.cache_key }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - id: load-docker-cache
         name: Load Docker Cache
         uses: actions/cache@v3

.github/workflows/create_release.yml

@@ -11,7 +11,7 @@ jobs:
     name: Generate git tags
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: '0'
       - uses: actions/github-script@v6
@@ -60,7 +60,7 @@ jobs:
       - generate-tags
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: "0"
 
@@ -95,7 +95,7 @@ jobs:
           ]
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: "0"
 
@@ -132,7 +132,7 @@ jobs:
       - build-docker-images
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: "0"
 
@@ -165,7 +165,7 @@ jobs:
       - build-docker-images
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: "0"
 

.github/workflows/end2end.yml

@@ -24,8 +24,8 @@ jobs:
           FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)"
           echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV}
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: unionai/[email protected].1
+        uses: actions/checkout@v4
+      - uses: unionai/[email protected].3
         name: Setup flytectl
       - uses: actions/setup-python@v3
         with:
@@ -71,7 +71,7 @@ jobs:
           pip freeze
       - name: Checkout flytesnacks
         if: ${{ inputs.priorities == 'P0' }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: flyteorg/flytesnacks
           path: flytesnacks

.github/workflows/flytectl-install.yml

@@ -0,0 +1,33 @@
+name: Flytectl-specific checks
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    paths:
+      - flytectl/**
+  push:
+    branches:
+      - master
+
+jobs:
+  install-script:
+    name: Install script
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - v0.8.20
+          - latest
+          # Test the case where no version is specified
+          - " "
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+          chmod +x ./flytectl/install.sh
+          ./flytectl/install.sh ${{ matrix.version }}
+          ./bin/flytectl version
+

.github/workflows/flytectl-release.yml

@@ -0,0 +1,27 @@
+name: Flytectl release
+
+on:
+  push:
+    tags:
+      - flytectl/v*.*.*
+
+jobs:
+  release:
+    name: Goreleaser
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: "0"
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.21"
+      - name: Run GoReleaser
+        uses: goreleaser/goreleaser-action@v5
+        with:
+          distribution: goreleaser-pro
+          args: release --rm-dist -f flytectl/.goreleaser.yml
+        env:
+          GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
+          GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }}

.github/workflows/flyteidl-buf-publish.yml

@@ -12,7 +12,7 @@ jobs:
   buf:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: bufbuild/buf-setup-action@v1
       - uses: bufbuild/buf-push-action@v1
         with:

.github/workflows/flyteidl-checks.yml

@@ -39,7 +39,7 @@ jobs:
       component: flyteidl
       go-version: ${{ needs.unpack-envvars.outputs.go-version }}
     secrets:
-      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   generate:
     name: Check Go Generate
     needs:

.github/workflows/flyteidl-release.yml

@@ -2,16 +2,17 @@ name: Upload flyteidl to PyPI and npm
 
 on:
   release:
-    types: [created]
+    types: [published]
 
 jobs:
   deploy-to-pypi:
+    if: "!startsWith(github.event.release.tag_name, 'flytectl/')"
     runs-on: ubuntu-latest
     defaults:
       run:
         working-directory: flyteidl
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python
         uses: actions/setup-python@v1
         with:
@@ -28,12 +29,13 @@ jobs:
           python -m build
           twine upload dist/*
   deploy-to-npm:
+    if: "!startsWith(github.event.release.tag_name, 'flytectl/')"
     runs-on: ubuntu-latest
     defaults:
       run:
         working-directory: flyteidl
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: actions/setup-node@v1
         with:
           node-version: "12.x"

.github/workflows/generate_flyte_manifest.yml

@@ -12,10 +12,10 @@ jobs:
     name: Update Flyte components
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: "0"
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v4
         with:
           go-version: "1.21"
       - name: Update references