Merge branch 'master' into update-pyflyte-serve-agent-command

# Conflicts: # deployment/agent/flyte_agent_helm_generated.yaml
flyteorg · Jan 5, 2024 · 555f62b · 555f62b
2 parents 0e2ef95 + b96419c
commit 555f62b
Show file tree

Hide file tree

Showing 412 changed files with 135,267 additions and 50,751 deletions.
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -10,7 +10,7 @@ on:
     branches:
       - master
 env:
-  GO_VERSION: "1.19"
+  GO_VERSION: "1.21"
   PRIORITIES: "P0"
 jobs:
   unpack-envvars:

diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml
@@ -25,4 +25,4 @@ jobs:
       - name: Codespell
         uses: codespell-project/actions-codespell@v2
         with:
-          skip: "*.pb"
+          skip: "*.pb,monodocs-environment.lock.yaml"
diff --git a/.github/workflows/flyteidl-buf-publish.yml b/.github/workflows/flyteidl-buf-publish.yml
@@ -3,6 +3,7 @@ name: Publish flyteidl Buf Package
 on:
   push:
     branches:
+      - artifacts-shell
       - artifacts
       - master
     paths:

diff --git a/.github/workflows/flyteidl-checks.yml b/.github/workflows/flyteidl-checks.yml
@@ -10,7 +10,7 @@ on:
     branches:
       - master
 env:
-  GO_VERSION: "1.19"
+  GO_VERSION: "1.21"
 jobs:
   unpack-envvars:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/generate_flyte_manifest.yml b/.github/workflows/generate_flyte_manifest.yml
@@ -17,7 +17,7 @@ jobs:
           fetch-depth: "0"
       - uses: actions/setup-go@v2
         with:
-          go-version: "1.19"
+          go-version: "1.21"
       - name: Update references
         env:
           VERSION: ${{ github.event.inputs.next-version }}

diff --git a/.github/workflows/go_generate.yml b/.github/workflows/go_generate.yml
@@ -25,6 +25,8 @@ jobs:
         with:
           repo-token: ${{ secrets.FLYTE_BOT_PAT }}
       - uses: bufbuild/buf-setup-action@v1
+        with:
+          github_token: ${{ secrets.FLYTE_BOT_PAT }}
       - uses: actions/setup-go@v3
         with:
           go-version: ${{ inputs.go-version }}

diff --git a/.github/workflows/single-binary.yml b/.github/workflows/single-binary.yml
@@ -6,14 +6,6 @@ concurrency:
 
 on:
   pull_request:
-    paths:
-      - .github/workflows/single-binary.yml
-      - charts/flyte-binary/**
-      - charts/flyte-sandbox/**
-      - cmd/**
-      - docker/sandbox-bundled/**
-      - Dockerfile
-      - go.*
   push:
     branches:
       - master
@@ -27,7 +19,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: "1.19"
+          go-version: "1.21"
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
@@ -131,7 +123,7 @@ jobs:
     env:
       FLYTESNACKS_PRIORITIES: "P0"
       FLYTESNACKS_VERSION: ""
-    timeout-minutes: 20
+    timeout-minutes: 60
     needs: [build-and-push-single-binary-image]
     steps:
       - name: Set latest Flytesnacks release

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -18,7 +18,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: "1.19"
+          go-version: "1.21"
       - name: Compile
         run: make compile
       - name: Run tests
@@ -31,14 +31,15 @@ jobs:
         uses: actions/checkout@v4
       - uses: conda-incubator/setup-miniconda@v3
         with:
+          auto-update-conda: true
           python-version: 3.9
-          mamba-version: "*"
-          channels: conda-forge
-          channel-priority: true
-          activate-environment: monodocs-env
-          environment-file: monodocs-environment.yaml
       - shell: bash -el {0}
         run: |
+          conda install -c conda-forge conda-lock
+          conda-lock install -n monodocs-env monodocs-environment.lock.yaml
+      - shell: bash -el {0}
+        run: |
+          conda activate monodocs-env
           conda info
           conda list
           conda config --show-sources
@@ -48,15 +49,17 @@ jobs:
         uses: ts-graphviz/setup-graphviz@v1
       - name: Build the documentation
         shell: bash -el {0}
-        run: make docs
+        run: |
+          conda activate monodocs-env
+          make docs
 
   generate_kustomize:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: "1.19"
+          go-version: "1.21"
       - name: Kustomize and diff
         run: DELTA_CHECK=true make kustomize
 
@@ -66,6 +69,6 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: "1.19"
+          go-version: "1.21"
       - name: Helm and diff
         run: DELTA_CHECK=true make helm
diff --git a/Dockerfile b/Dockerfile
@@ -2,7 +2,7 @@ ARG FLYTECONSOLE_VERSION=latest
 FROM ghcr.io/flyteorg/flyteconsole:${FLYTECONSOLE_VERSION} AS flyteconsole
 
 
-FROM --platform=${BUILDPLATFORM} golang:1.19.13-bookworm AS flytebuilder
+FROM --platform=${BUILDPLATFORM} golang:1.21.5-bookworm AS flytebuilder
 
 ARG TARGETARCH
 ENV GOARCH "${TARGETARCH}"

diff --git a/Dockerfile.datacatalog b/Dockerfile.datacatalog
@@ -3,7 +3,7 @@
 # 
 # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.19-alpine3.16 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.21.5-alpine3.18 as builder
 
 ARG TARGETARCH
 ENV GOARCH "${TARGETARCH}"
@@ -39,7 +39,7 @@ RUN make linux_compile
 ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
-FROM alpine:3.16
+FROM alpine:3.18
 LABEL org.opencontainers.image.source=https://github.com/flyteorg/datacatalog
 
 COPY --from=builder /artifacts /bin

diff --git a/Dockerfile.flyteadmin b/Dockerfile.flyteadmin
@@ -3,7 +3,7 @@
 # 
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.19-alpine3.16 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.21.5-alpine3.18 as builder
 
 ARG TARGETARCH
 ENV GOARCH "${TARGETARCH}"
@@ -39,7 +39,7 @@ RUN make linux_compile
 ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
-FROM alpine:3.16
+FROM alpine:3.18
 LABEL org.opencontainers.image.source https://github.com/flyteorg/flyteadmin
 
 COPY --from=builder /artifacts /bin

diff --git a/Dockerfile.flytecopilot b/Dockerfile.flytecopilot
@@ -3,7 +3,7 @@
 # 
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.19-alpine3.16 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.21.5-alpine3.18 as builder
 
 ARG TARGETARCH
 ENV GOARCH "${TARGETARCH}"
@@ -29,7 +29,7 @@ RUN make linux_compile
 ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
-FROM alpine:3.16
+FROM alpine:3.18
 LABEL org.opencontainers.image.source https://github.com/lyft/flyteplugins
 
 COPY --from=builder /artifacts /bin

diff --git a/Dockerfile.flytepropeller b/Dockerfile.flytepropeller
@@ -3,7 +3,7 @@
 # 
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.19-alpine3.16 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.21.5-alpine3.18 as builder
 
 ARG TARGETARCH
 ENV GOARCH "${TARGETARCH}"
@@ -29,7 +29,7 @@ RUN make linux_compile
 ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
-FROM alpine:3.16
+FROM alpine:3.18
 LABEL org.opencontainers.image.source https://github.com/flyteorg/flytepropeller
 
 COPY --from=builder /artifacts /bin

diff --git a/Dockerfile.flytescheduler b/Dockerfile.flytescheduler
@@ -3,7 +3,7 @@
 # 
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.19-alpine3.16 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.21.5-alpine3.18 as builder
 
 ARG TARGETARCH
 ENV GOARCH "${TARGETARCH}"
@@ -32,7 +32,7 @@ RUN make linux_compile_scheduler
 ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
-FROM alpine:3.15
+FROM alpine:3.18
 LABEL org.opencontainers.image.source https://github.com/flyteorg/flyteadmin
 
 COPY --from=builder /artifacts /bin

diff --git a/Makefile b/Makefile
@@ -57,6 +57,14 @@ install-piptools: ## Install pip-tools
 doc-requirements.txt: doc-requirements.in install-piptools
 	$(call PIP_COMPILE,doc-requirements.in)
 
+.PHONY: install-conda-lock
+install-conda-lock:
+	pip install conda-lock
+
+.PHONY: conda-lock
+conda-lock: install-conda-lock
+	conda-lock -f monodocs-environment.yaml --without-cuda --lockfile monodocs-environment.lock.yaml
+
 .PHONY: stats
 stats:
 	@generate-dashboard -o deployment/stats/prometheus/flytepropeller-dashboard.json stats/flytepropeller.dashboard.py
-Original file line number
+Diff line change
@@ Expand Up / @@ -3,6 +3,7 @@ name: Publish flyteidl Buf Package @@
     on:
       push:
         branches:
+          - artifacts-shell
           - artifacts
           - master
         paths:
@@ Expand Down @@