Stop publishing dind and latest tag for sandbox (flyteorg#2085)

* Stop publishing dind and latest tag for sandbox Signed-off-by: Yuvraj <[email protected]> * refactor docker sandbox Signed-off-by: Yuvraj <[email protected]> * More changes Signed-off-by: Yuvraj <[email protected]> * Added release tag in sandbox image Signed-off-by: Yuvraj <[email protected]> * more changes Signed-off-by: Yuvraj <[email protected]>
Tom-Newton · Feb 4, 2022 · 5bf853a · 5bf853a
1 parent 4f8f94e
commit 5bf853a
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 185 deletions.
diff --git a/.github/workflows/sandbox.yml b/.github/workflows/sandbox.yml
@@ -10,70 +10,6 @@ on:
     types: [published]
 
 jobs:
-  sandbox-build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: "0"
-      - name: Set flyte version to release
-        id: set_version
-        run: |
-          if [ ${{ github.event_name}} = "release" ]; then
-            echo ::set-output name=flyte_version::$(echo ${{ github.event.release.tag_name }})
-          else
-            echo ::set-output name=flyte_version::latest
-          fi
-      - name: Prepare sandbox Image Names
-        id: sandbox-names
-        uses: docker/metadata-action@v3
-        with:
-          # list of Docker images to use as base name for tags
-          images: |
-            ghcr.io/${{ github.repository_owner }}/flyte-sandbox
-          tags: |
-            latest
-            type=sha,format=long
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Cache Docker layers
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-single-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-single-buildx
-      - name: Login to GitHub Container Registry
-        if: ${{ github.event_name == 'release' }}
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: "${{ secrets.FLYTE_BOT_USERNAME }}"
-          password: "${{ secrets.FLYTE_BOT_PAT }}"
-      - name: Build and push Sandbox image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          platforms: linux/arm64, linux/amd64
-          push: ${{ github.event_name == 'release' }}
-          target: default
-          build-args: "FLYTE_VERSION=${{ steps.set_version.outputs.flyte_version }}"
-          tags: ${{ steps.sandbox-names.outputs.tags }}
-          file: docker/sandbox/Dockerfile
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new
-      - # Temp fix
-        # https://github.com/docker/build-push-action/issues/252
-        # https://github.com/moby/buildkit/issues/1896
-        name: Move cache
-        run: |
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
-
   sandbox-build-dind:
     runs-on: ubuntu-latest
     steps:
@@ -98,6 +34,7 @@ jobs:
             ghcr.io/${{ github.repository_owner }}/flyte-sandbox
           tags: |
             dind
+            ${{ steps.set_version.outputs.flyte_version }}
             type=sha,format=long, prefix=dind-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1

diff --git a/docker/sandbox/Dockerfile b/docker/sandbox/Dockerfile
@@ -1,17 +1,3 @@
-FROM golang:1.16.5-alpine3.13 AS go_builder_
-
-# Install dependencies
-RUN apk add --no-cache build-base git make
-
-# Create directory to store built artifacts
-ARG INSTALL_DIR="/install"
-RUN mkdir -p ${INSTALL_DIR}
-
-ARG BUILDKIT_CLI_FOR_KUBECTL_VERSION="v0.1.2"
-RUN git clone -b ${BUILDKIT_CLI_FOR_KUBECTL_VERSION} --single-branch --depth 1 https://github.com/vmware-tanzu/buildkit-cli-for-kubectl.git ${GOPATH}/src/github.com/vmware-tanzu/buildkit-cli-for-kubectl \
-    && make -C ${GOPATH}/src/github.com/vmware-tanzu/buildkit-cli-for-kubectl ${INSTALL_DIR}/linux/kubectl-build BIN_DIR=${INSTALL_DIR} VERSION=${BUILDKIT_CLI_FOR_KUBECTL_VERSION}
-
-
 FROM alpine:3.13.5 AS base_
 
 # Install dependencies
@@ -46,47 +32,13 @@ RUN wget -q -O /flyteorg/bin/get_helm.sh https://raw.githubusercontent.com/helm/
 # Install flytectl
 RUN wget -q -O - https://raw.githubusercontent.com/flyteorg/flytectl/master/install.sh | BINDIR=/flyteorg/bin sh -s
 
-# Install buildkit-cli-for-kubectl
-COPY --from=go_builder_ /install/linux/ /flyteorg/bin/
-
 # Copy flyte chart
 COPY charts/flyte/ /flyteorg/share/flyte
 COPY charts/flyte-core/ /flyteorg/share/flyte-core
 
 # Copy scripts
 COPY docker/sandbox/kubectl docker/sandbox/cgroup-v2-hack.sh docker/sandbox/wait-for-flyte.sh /flyteorg/bin/
 
-
-FROM base_ AS default
-
-# Install dependencies
-RUN apk add --no-cache bash git make tini curl jq
-
-# Copy entrypoints
-COPY docker/sandbox/flyte-entrypoint-default.sh /flyteorg/bin/flyte-entrypoint.sh
-COPY docker/sandbox/bashrc /root/.bashrc
-
-ARG FLYTE_VERSION="latest"
-ENV FLYTE_VERSION "${FLYTE_VERSION}"
-
-ARG FLYTE_TEST="release"
-ENV FLYTE_TEST "${FLYTE_TEST}"
-
-# Update PATH variable
-ENV PATH "/flyteorg/bin:${PATH}"
-
-# Declare volumes for k3s
-VOLUME /var/lib/kubelet
-VOLUME /var/lib/rancher/k3s
-VOLUME /var/lib/cni
-VOLUME /var/log
-
-# Expose Flyte ports
-EXPOSE 30081 30082 30084 30088
-
-ENTRYPOINT ["tini", "flyte-entrypoint.sh"]
-
-
 FROM docker:20.10.11-dind AS dind
 
 # Install dependencies
@@ -96,7 +48,7 @@ RUN apk add --no-cache bash git make tini curl jq
 COPY --from=base_ /flyteorg/ /flyteorg/
 
 # Copy entrypoints
-COPY docker/sandbox/flyte-entrypoint-dind.sh /flyteorg/bin/flyte-entrypoint.sh
+COPY docker/sandbox/flyte-entrypoint-default.sh /flyteorg/bin/flyte-entrypoint.sh
 
 ARG FLYTE_VERSION="latest"
 ENV FLYTE_VERSION "${FLYTE_VERSION}"

diff --git a/docker/sandbox/flyte-entrypoint-default.sh b/docker/sandbox/flyte-entrypoint-default.sh
@@ -7,10 +7,27 @@ cgroup-v2-hack.sh
 
 trap 'pkill -P $$' EXIT
 
+monitor() {
+    while : ; do
+        for pid in $@ ; do
+            kill -0 $pid &> /dev/null || exit 1
+        done
+
+        sleep 1
+    done
+}
+
+# Start docker daemon
+echo "Starting Docker daemon..."
+dockerd &> /var/log/dockerd.log &
+DOCKERD_PID=$!
+timeout 600 sh -c "until docker info &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for dockerd to start"; exit 1 )
+echo "Done."
+
 # Start k3s
 echo "Starting k3s cluster..."
 KUBERNETES_API_PORT=${KUBERNETES_API_PORT:-6443}
-k3s server --no-deploy=traefik --no-deploy=servicelb --no-deploy=local-storage --no-deploy=metrics-server --https-listen-port=${KUBERNETES_API_PORT} &> /var/log/k3s.log &
+k3s server --docker --no-deploy=traefik --no-deploy=servicelb --no-deploy=local-storage --no-deploy=metrics-server --https-listen-port=${KUBERNETES_API_PORT} &> /var/log/k3s.log &
 K3S_PID=$!
 timeout 600 sh -c "until k3s kubectl explain deployment &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for the Kubernetes cluster to start"; exit 1 )
 echo "Done."
@@ -23,7 +40,6 @@ fi
 
 # Deploy flyte
 echo "Deploying Flyte..."
-
 version=""
 charts="/flyteorg/share/flyte"
 
@@ -44,4 +60,10 @@ helm upgrade -n flyte --create-namespace flyte $charts --kubeconfig /etc/rancher
 
 wait-for-flyte.sh
 
-wait ${K3S_PID}
+# With flytectl sandbox --source flag, we mount the root volume to user source dir that will create helm & k8s cache specific directory.
+# In Linux, These file belongs to root user that is different then current user
+# In this case during fast serialization, Pyflyte will through error because of permission denied
+rm -rf /root/.cache /root/.kube /root/.config
+
+# Monitor running processes. Exit when the first process exits.
+monitor ${DOCKERD_PID} ${K3S_PID}
diff --git a/docker/sandbox/flyte-entrypoint-dind.sh b/docker/sandbox/flyte-entrypoint-dind.sh