New CI/CD for sandbox bundled and flyte single binary images (flyteor…

…g#2994)

Signed-off-by: Yuvraj <[email protected]>

.github/workflows/create_release.yml

@@ -34,12 +34,12 @@ jobs:
       matrix:
         component:
           [
-            flyteconsole,
-            flytepropeller,
-            flyteadmin,
-            datacatalog,
-            flytescheduler,
-            flytecopilot,
+              flyteconsole,
+              flytepropeller,
+              flyteadmin,
+              datacatalog,
+              flytescheduler,
+              flytecopilot,
           ]
     steps:
       - name: Checkout

.github/workflows/sandbox.yml

@@ -10,7 +10,7 @@ on:
     types: [published]
 
 jobs:
-  sandbox-build-dind:
+  trigger-sandbox-build:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -60,13 +60,23 @@ jobs:
           file: docker/sandbox/Dockerfile
           cache-from: type=gha
           cache-to: type=gha,mode=max
-  sandbox-lite-build-dind:
+
+  trigger-sandbox-lite-build:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v2
         with:
           fetch-depth: "0"
+      - name: Setup Golang caches
+        uses: actions/cache@v3
+        with:
+          path: |
+            /root/.cache/go-build
+            /root/go/pkg/mod
+          key: ${{ runner.os }}-golang-${{ hashFiles('go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-golang-
       - name: Set flyte version to release
         id: set_version
         run: |
@@ -109,3 +119,149 @@ jobs:
           file: docker/sandbox-lite/Dockerfile
           cache-from: type=gha
           cache-to: type=gha,mode=max
+
+
+  trigger-single-binary-build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - name: Setup Golang caches
+        uses: actions/cache@v3
+        with:
+          path: |
+            /root/.cache/go-build
+            /root/go/pkg/mod
+          key: ${{ runner.os }}-golang-${{ hashFiles('go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-golang-
+      - name: Set flyte version to release
+        id: set_version
+        run: |
+          if [ ${{ github.event_name}} = "release" ]; then
+            echo ::set-output name=flyte_version::$(echo ${{ github.event.release.tag_name }})
+          else
+            echo ::set-output name=flyte_version::latest
+          fi
+      - name: Prepare DIND Image Names
+        id: dind-names
+        uses: docker/metadata-action@v3
+        with:
+          # list of Docker images to use as base name for tags
+          images: |
+            ghcr.io/${{ github.repository_owner }}/flyte-binary
+          tags: |
+            ${{ inputs.version }}
+            type=sha,format=long,
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: "${{ secrets.FLYTE_BOT_USERNAME }}"
+          password: "${{ secrets.FLYTE_BOT_PAT }}"
+      - name: Export ARM64 Image
+        uses: docker/build-push-action@v3
+        env:
+          DOCKER_CLI_EXPERIMENTAL: enabled
+        with:
+          context: .
+          platforms: linux/arm64
+          tags: flyte-binary:sandbox
+          build-args: ""
+          file: Dockerfile
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          outputs: type=docker,dest=/tmp/flyte-arm64.tar
+      - name: Export AMD64 Image
+        uses: docker/build-push-action@v3
+        env:
+          DOCKER_CLI_EXPERIMENTAL: enabled
+        with:
+          context: .
+          platforms: linux/amd64
+          tags: flyte-binary:sandbox
+          build-args: ""
+          file: Dockerfile
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          outputs: type=docker,dest=/tmp/flyte-amd64.tar
+      - name: Build and push Image
+        if: ${{ github.event_name == 'release' }}
+        uses: docker/build-push-action@v3
+        env:
+          DOCKER_CLI_EXPERIMENTAL: enabled
+        with:
+          context: .
+          platforms: linux/arm64, linux/amd64
+          tags: ${{ steps.dind-names.outputs.tags }}
+          build-args: ""
+          file: Dockerfile
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          push: ${{ github.event_name == 'release' }}
+      - name: Upload Artifact GitHub Action
+        uses: actions/upload-artifact@v2
+        with:
+          name: single-binary-image
+          path: /tmp/flyte-*.tar
+
+
+  trigger-sandbox-bundled-build:
+    runs-on: ubuntu-latest
+    needs: [trigger-single-binary-build]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - uses: actions/download-artifact@v3
+        with:
+          name: single-binary-image
+          path: /tmp/
+      - name: Prepare DIND Image Names
+        id: dind-names
+        uses: docker/metadata-action@v3
+        with:
+          # list of Docker images to use as base name for tags
+          images: |
+            ghcr.io/${{ github.repository_owner }}/flyte-sandbox-bundled
+          tags: |
+            ${{ inputs.version }}
+            type=sha,format=long,
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          driver-opts: image=moby/buildkit:master
+          buildkitd-flags: "--allow-insecure-entitlement security.insecure"
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: "${{ secrets.FLYTE_BOT_USERNAME }}"
+          password: "${{ secrets.FLYTE_BOT_PAT }}"
+      - name: Prepare Sandbox images for single binary
+        run: |
+          make -C docker/sandbox-bundled images-ci
+      - name: Build and push Image
+        uses: docker/build-push-action@v3
+        env:
+          DOCKER_CLI_EXPERIMENTAL: enabled
+        with:
+          context: docker/sandbox-bundled
+          allow: "security.insecure"
+          platforms: linux/arm64, linux/amd64
+          tags: ${{ steps.dind-names.outputs.tags }}
+          file: docker/sandbox-bundled/images/dockerfiles/k3s/Dockerfile
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          push: true

.gitignore

@@ -24,11 +24,9 @@ __pycache__/
 /docker/sandbox-lite/storage/charts
 /docker/sandbox-lite/storage/Chart.lock
 **/*.bak-
-bin/-
-bin/
 .python-version
 dist/
 dist
 *.db
-flyte
 vendor/
+/docker/sandbox-bundled/images/tar

Dockerfile

@@ -0,0 +1,22 @@
+ARG FLYTE_VERSION="latest"
+
+FROM ghcr.io/flyteorg/flyteconsole-release:$FLYTE_VERSION AS flyteconsole
+
+
+FROM golang:1.19.1-bullseye AS flytebuilder
+
+ARG FLYTE_VERSION="master"
+
+WORKDIR /flyteorg/build
+RUN git clone --depth=1 https://github.com/flyteorg/flyte.git ./flyte -b $FLYTE_VERSION
+WORKDIR /flyteorg/build/flyte
+RUN go mod download
+COPY --from=flyteconsole /app/dist cmd/single/dist
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/go/pkg/mod \
+    go build -tags console -v -o dist/flyte cmd/main.go
+
+
+FROM gcr.io/distroless/base-debian11
+
+COPY --from=flytebuilder /flyteorg/build/flyte/dist/flyte /flyte
+ENTRYPOINT [ "/flyte" ]

docker/sandbox-bundled/Makefile

@@ -0,0 +1,67 @@
+.SILENT: help
+.PHONY: help
+help:
+	echo Available recipes:
+	cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' | awk 'BEGIN { FS = ":.*?## " } { cnt++; a[cnt] = $$1; b[cnt] = $$2; if (length($$1) > max) max = length($$1) } END { for (i = 1; i <= cnt; i++) printf "  $(shell tput setaf 6)%-*s$(shell tput setaf 0) %s\n", max, a[i], b[i] }'
+	tput sgr0
+
+export DOCKER_BUILDKIT := 1
+
+.PHONY: images
+images:
+	mkdir -p docker/sandbox-bundled/images/dockerfiles/tar/arm64
+	mkdir -p docker/sandbox-bundled/images/dockerfiles/tar/amd64
+	docker pull ghcr.io/flyteorg/flyte-binary:$(VERSION) --platform linux/arm64
+	docker tag ghcr.io/flyteorg/flyte-binary:$(VERSION) ghcr.io/flyteorg/flyte-binary:latest
+	docker save --output  docker/sandbox-bundled/images/dockerfiles/tar/arm64/flyte.tar ghcr.io/flyteorg/flyte-binary:latest
+	docker rmi ghcr.io/flyteorg/flyte-binary
+	docker pull ghcr.io/flyteorg/flyte-binary:$(VERSION) --platform linux/amd64
+	docker tag ghcr.io/flyteorg/flyte-binary:$(VERSION) ghcr.io/flyteorg/flyte-binary:latest
+	docker save --output  docker/sandbox-bundled/images/dockerfiles/tar/amd64/flyte.tar ghcr.io/flyteorg/flyte-binary:latest
+
+.PHONY: images-ci
+images-ci:
+	mkdir -p images/dockerfiles/tar/arm64
+	mkdir -p images/dockerfiles/tar/amd64
+	cp /tmp/flyte-amd64.tar images/dockerfiles/tar/amd64/flyte.tar
+	cp /tmp/flyte-arm64.tar images/dockerfiles/tar/arm64/flyte.tar
+
+.PHONY: clean
+clean: ## Cleanup state
+	rm -rf data
+
+.PHONY: build
+build:  images
+	docker build -t flyte-sandbox -f images/dockerfiles/k3s/Dockerfile .
+
+data/flyte.yaml:
+	mkdir -p $(shell dirname $@)
+	cp bootstrap/flyte/flyte.yaml $@
+
+.PHONY: start
+start:  ## Start sandbox
+	[ -n "$(shell docker ps --filter name=^flyte-sandbox$$ --format {{.Names}})" ] || \
+		docker run --detach --rm --privileged --name flyte-sandbox --volume $(PWD)/data:/srv/flyte --env K3S_KUBECONFIG_OUTPUT=/srv/flyte/kubeconfig --publish "6443" --publish "30000:30000" --publish "30001:30001" --publish "30002:30002" --publish "30080:30080" docker.io/library/flyte-sandbox
+
+.PHONY: kubeconfig
+.SILENT: kubeconfig
+kubeconfig: ## Set up kubeconfig
+	sed -i -e "/server:/ s/: .*/: https:\/\/127.0.0.1:$(shell docker port flyte-sandbox | grep ^6443 | awk '{print $$3}' | awk -F: '{print $$2}')/" data/kubeconfig
+	echo "export KUBECONFIG=$(PWD)/data/kubeconfig"
+
+.PHONY: stop
+stop: ## Stop sandbox
+	docker stop flyte-sandbox
+
+.PHONY: console
+console: ## Open Flyte console in browser
+	open http://localhost:30080/console
+
+.venv:
+	python -m venv .venv
+	. .venv/bin/activate && pip install flytekit
+
+.PHONY: example
+example: export FLYTECTL_CONFIG := example/config.yaml
+example: .venv ## Run example script
+	. .venv/bin/activate && pyflyte run --remote example/example.py wf --n 500 --mean 42 --sigma 2

docker/sandbox-bundled/README.md

@@ -0,0 +1,11 @@
+# Flyte Deployment Sandbox
+
+First make images
+```
+ytong@Yees-MBP:~/go/src/github.com/flyteorg/flyte/docker/sandbox-ultra [flyte-sandbox] (cicd-sandbox-lite) $ make images
+```
+
+then build the k3s image.
+```
+ytong@Yees-MBP:~/go/src/github.com/flyteorg/flyte/docker/sandbox-ultra [] (cicd-sandbox-lite) $ docker buildx build --file images/dockerfiles/k3s/Dockerfile --platform linux/arm64,linux/amd64 --push --tag ghcr.io/flyteorg/flyte-sandbox-lite:ultra7 .
+```