Tomnewton/dashboard updates

.gitattributes

@@ -1 +1,5 @@
 docs/**/*html linguist-generated=true
+flyteidl/gen/** linguist-generated=true
+flyteidl/protos/**/*.rst linguist-generated=true
+flyteidl/clients/go/assets/admin.swagger.json linguist-generated=true
+

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,11 +1,13 @@
 ## Tracking issue
-_https://github.com/flyteorg/flyte/issues/<number>_
-
-<!-- If your PR fixes an open issue, use `Closes #999` to link your PR with the issue. #999 stands for the issue number you are fixing -->
+<!--
+If your PR fixes an open issue, use `Closes #999` to link your PR with the issue.
+Example: Closes #999
 
-<!-- Remove this section if not applicable -->
+If your PR is related to an issue or PR, use `Related to #999` to link your PR.
+Example: Related to #999
 
-<!-- Example: Closes #31 -->
+Remove this section if not applicable
+-->
 
 ## Why are the changes needed?
 

.github/workflows/checks.yml

@@ -34,6 +34,7 @@ jobs:
           - flyteadmin
           # TODO(monorepo): Enable lint flytecopilot
           # - flytecopilot
+          - flytectl
           - flyteidl
           - flyteplugins
           - flytepropeller
@@ -53,6 +54,7 @@ jobs:
           - datacatalog
           - flyteadmin
           - flytecopilot
+          - flytectl
           - flyteplugins
           - flytepropeller
           - flytestdlib
@@ -61,35 +63,20 @@ jobs:
       component: ${{ matrix.component }}
       go-version: ${{ needs.unpack-envvars.outputs.go-version }}
     secrets:
-      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
-  docker-build:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  flyteadmin-int-tests-image-build:
     strategy:
       fail-fast: false
-      matrix:
-        component:
-          - datacatalog
-          - flyteadmin
-          - flytecopilot
-          - flytepropeller
-    name: Docker Build Images
+    name: flyteadmin integration tests image 
     uses: ./.github/workflows/component_docker_build.yml
     with:
-       component: ${{ matrix.component }}
-
-  # TODO(monorepo): these tests are broken. They never test an actual change.
-  # endtoend:
-  #   name: End2End Test
-  #   needs: [ docker-build ]
-  #   uses: ./.github/workflows/end2end.yml
-  #   with:
-  #     # Reusing the output of the matrix is ok as they are essentially writing the same value (i.e. the directory artifacts are written + run id)
-  #     cache_key: ${{ needs.docker-build.outputs.cache_key }}
-  #     priorities: "P0"
+      component: flyteadmin
 
-  integration:
+  flyteadmin-integration-tests:
     name: Integration Test
     needs:
-      - docker-build
+      - flyteadmin-int-tests-image-build
       - unpack-envvars
     strategy:
       fail-fast: false
@@ -99,7 +86,7 @@ jobs:
     uses: ./.github/workflows/integration.yml
     with:
       component: ${{ matrix.component }}
-      cache_key: ${{ needs.docker-build.outputs.cache_key }}
+      cache_key: ${{ needs.flyteadmin-int-tests-image-build.outputs.cache_key }}
       go-version: ${{ needs.unpack-envvars.outputs.go-version }}
 
   generate:
@@ -113,6 +100,7 @@ jobs:
           - datacatalog
           - flyteadmin
           - flytecopilot
+          - flytectl
           - flytepropeller
     uses: ./.github/workflows/go_generate.yml
     with:

.github/workflows/codespell.yml

@@ -21,8 +21,9 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Codespell
         uses: codespell-project/actions-codespell@v2
         with:
           skip: "*.pb,monodocs-environment.lock.yaml"
+          ignore_words_list: crate,lightyear,fo

.github/workflows/component_docker_build.yml

@@ -17,7 +17,7 @@ jobs:
       cache_key: ${{ steps.cache_key.outputs.cache_key }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - id: load-docker-cache
         name: Load Docker Cache
         uses: actions/cache@v3

.github/workflows/create_release.yml

@@ -11,7 +11,7 @@ jobs:
     name: Generate git tags
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: '0'
       - uses: actions/github-script@v6
@@ -60,7 +60,7 @@ jobs:
       - generate-tags
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: "0"
 
@@ -95,7 +95,7 @@ jobs:
           ]
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: "0"
 
@@ -132,7 +132,7 @@ jobs:
       - build-docker-images
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: "0"
 
@@ -165,7 +165,7 @@ jobs:
       - build-docker-images
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: "0"
 

.github/workflows/end2end.yml

@@ -24,7 +24,7 @@ jobs:
           FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)"
           echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV}
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - uses: unionai/[email protected]
         name: Setup flytectl
       - uses: actions/setup-python@v3
@@ -71,7 +71,7 @@ jobs:
           pip freeze
       - name: Checkout flytesnacks
         if: ${{ inputs.priorities == 'P0' }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: flyteorg/flytesnacks
           path: flytesnacks

.github/workflows/flyteidl-buf-publish.yml

@@ -12,7 +12,7 @@ jobs:
   buf:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: bufbuild/buf-setup-action@v1
       - uses: bufbuild/buf-push-action@v1
         with:

.github/workflows/flyteidl-checks.yml

@@ -39,7 +39,7 @@ jobs:
       component: flyteidl
       go-version: ${{ needs.unpack-envvars.outputs.go-version }}
     secrets:
-      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   generate:
     name: Check Go Generate
     needs:
@@ -50,3 +50,5 @@ jobs:
     with:
       component: flyteidl
       go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}

.github/workflows/flyteidl-release.yml

@@ -2,7 +2,7 @@ name: Upload flyteidl to PyPI and npm
 
 on:
   release:
-    types: [created]
+    types: [published]
 
 jobs:
   deploy-to-pypi:
@@ -11,34 +11,29 @@ jobs:
       run:
         working-directory: flyteidl
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python
         uses: actions/setup-python@v1
         with:
           python-version: "3.x"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install setuptools wheel twine
-      - name: Set version in python package
-        run: |
-          # from refs/tags/v1.2.3 get 1.2.3
-          VERSION=$(echo $GITHUB_REF | sed 's#.*/v##')
-          VERSION=$VERSION make update_pyversion
-        shell: bash
+          pip install build twine
       - name: Build and publish
         env:
           TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
         run: |
-          python setup.py sdist bdist_wheel
+          python -m build
           twine upload dist/*
   deploy-to-npm:
     runs-on: ubuntu-latest
     defaults:
       run:
         working-directory: flyteidl
     steps:
+      - uses: actions/checkout@v4
       - uses: actions/setup-node@v1
         with:
           node-version: "12.x"

.github/workflows/generate_flyte_manifest.yml

@@ -12,18 +12,17 @@ jobs:
     name: Update Flyte components
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: "0"
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v4
         with:
           go-version: "1.21"
       - name: Update references
         env:
           VERSION: ${{ github.event.inputs.next-version }}
         run: |
           make release_automation
-          make kustomize
           make helm
       - name: Create Pull Request
         id: cpr
@@ -39,17 +38,12 @@ jobs:
           title: 'Update Flyte components'
           body: |
             Updated flyte deployment
-            - Updated GCP Flyte kustomize generated manifest file
-            - Updated EKS Flyte kustomize generated manifest file
-            - Updated Sandbox Flyte  kustomize generated manifest file
-            - Updated TEST Flyte kustomize generated manifest file
             - Updated GCP Flyte helm generated manifest file
             - Updated EKS Flyte helm generated manifest file
             - Updated Sandbox Flyte helm generated manifest file
             - Updated TEST Flyte helm generated manifest file
             - Auto-generated by [flyte-bot]
           labels: |
-            kustomize
             helm
           team-reviewers: |
             flyte-maintainers

.github/workflows/go_generate.yml

@@ -20,14 +20,14 @@ jobs:
       run:
         working-directory: ${{ inputs.component }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: arduino/setup-protoc@v1
         with:
           repo-token: ${{ secrets.FLYTE_BOT_PAT }}
       - uses: bufbuild/buf-setup-action@v1
         with:
           github_token: ${{ secrets.FLYTE_BOT_PAT }}
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
         with:
           go-version: ${{ inputs.go-version }}
       - name: Go generate and diff

.github/workflows/integration.yml

@@ -22,7 +22,9 @@ jobs:
         working-directory: ${{ inputs.component }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: "0"
       - id: load-docker-cache
         name: Load Docker Cache
         uses: actions/cache@v3
@@ -35,7 +37,7 @@ jobs:
         with:
           version: "v0.11.1"
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: ${{ inputs.go-version }}
       - name: Integration

.github/workflows/lint.yml

@@ -18,9 +18,9 @@ jobs:
         working-directory: ${{ inputs.component }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: ${{ inputs.go-version }}
       - name: Lint