From 5bf853a6ce8dde82507de4b0dce8f298a77ae346 Mon Sep 17 00:00:00 2001 From: Yuvraj Date: Sat, 5 Feb 2022 01:31:20 +0530 Subject: [PATCH] Stop publishing dind and latest tag for sandbox (#2085) * Stop publishing dind and latest tag for sandbox Signed-off-by: Yuvraj * refactor docker sandbox Signed-off-by: Yuvraj * More changes Signed-off-by: Yuvraj * Added release tag in sandbox image Signed-off-by: Yuvraj * more changes Signed-off-by: Yuvraj --- .github/workflows/sandbox.yml | 65 +------------------- docker/sandbox/Dockerfile | 50 +--------------- docker/sandbox/flyte-entrypoint-default.sh | 28 ++++++++- docker/sandbox/flyte-entrypoint-dind.sh | 69 ---------------------- 4 files changed, 27 insertions(+), 185 deletions(-) delete mode 100755 docker/sandbox/flyte-entrypoint-dind.sh diff --git a/.github/workflows/sandbox.yml b/.github/workflows/sandbox.yml index 11c7e1d824..d126086c1f 100644 --- a/.github/workflows/sandbox.yml +++ b/.github/workflows/sandbox.yml @@ -10,70 +10,6 @@ on: types: [published] jobs: - sandbox-build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Set flyte version to release - id: set_version - run: | - if [ ${{ github.event_name}} = "release" ]; then - echo ::set-output name=flyte_version::$(echo ${{ github.event.release.tag_name }}) - else - echo ::set-output name=flyte_version::latest - fi - - name: Prepare sandbox Image Names - id: sandbox-names - uses: docker/metadata-action@v3 - with: - # list of Docker images to use as base name for tags - images: | - ghcr.io/${{ github.repository_owner }}/flyte-sandbox - tags: | - latest - type=sha,format=long - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - name: Cache Docker layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-single-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-single-buildx - - name: Login to GitHub Container Registry - if: ${{ github.event_name == 'release' }} - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - - name: Build and push Sandbox image - uses: docker/build-push-action@v2 - with: - context: . - platforms: linux/arm64, linux/amd64 - push: ${{ github.event_name == 'release' }} - target: default - build-args: "FLYTE_VERSION=${{ steps.set_version.outputs.flyte_version }}" - tags: ${{ steps.sandbox-names.outputs.tags }} - file: docker/sandbox/Dockerfile - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache - sandbox-build-dind: runs-on: ubuntu-latest steps: @@ -98,6 +34,7 @@ jobs: ghcr.io/${{ github.repository_owner }}/flyte-sandbox tags: | dind + ${{ steps.set_version.outputs.flyte_version }} type=sha,format=long, prefix=dind- - name: Set up QEMU uses: docker/setup-qemu-action@v1 diff --git a/docker/sandbox/Dockerfile b/docker/sandbox/Dockerfile index 93bb2894cb..9c8c0cb709 100644 --- a/docker/sandbox/Dockerfile +++ b/docker/sandbox/Dockerfile @@ -1,17 +1,3 @@ -FROM golang:1.16.5-alpine3.13 AS go_builder_ - -# Install dependencies -RUN apk add --no-cache build-base git make - -# Create directory to store built artifacts -ARG INSTALL_DIR="/install" -RUN mkdir -p ${INSTALL_DIR} - -ARG BUILDKIT_CLI_FOR_KUBECTL_VERSION="v0.1.2" -RUN git clone -b ${BUILDKIT_CLI_FOR_KUBECTL_VERSION} --single-branch --depth 1 https://github.com/vmware-tanzu/buildkit-cli-for-kubectl.git ${GOPATH}/src/github.com/vmware-tanzu/buildkit-cli-for-kubectl \ - && make -C ${GOPATH}/src/github.com/vmware-tanzu/buildkit-cli-for-kubectl ${INSTALL_DIR}/linux/kubectl-build BIN_DIR=${INSTALL_DIR} VERSION=${BUILDKIT_CLI_FOR_KUBECTL_VERSION} - - FROM alpine:3.13.5 AS base_ # Install dependencies @@ -46,9 +32,6 @@ RUN wget -q -O /flyteorg/bin/get_helm.sh https://raw.githubusercontent.com/helm/ # Install flytectl RUN wget -q -O - https://raw.githubusercontent.com/flyteorg/flytectl/master/install.sh | BINDIR=/flyteorg/bin sh -s -# Install buildkit-cli-for-kubectl -COPY --from=go_builder_ /install/linux/ /flyteorg/bin/ - # Copy flyte chart COPY charts/flyte/ /flyteorg/share/flyte COPY charts/flyte-core/ /flyteorg/share/flyte-core @@ -56,37 +39,6 @@ COPY charts/flyte-core/ /flyteorg/share/flyte-core # Copy scripts COPY docker/sandbox/kubectl docker/sandbox/cgroup-v2-hack.sh docker/sandbox/wait-for-flyte.sh /flyteorg/bin/ - -FROM base_ AS default - -# Install dependencies -RUN apk add --no-cache bash git make tini curl jq - -# Copy entrypoints -COPY docker/sandbox/flyte-entrypoint-default.sh /flyteorg/bin/flyte-entrypoint.sh -COPY docker/sandbox/bashrc /root/.bashrc - -ARG FLYTE_VERSION="latest" -ENV FLYTE_VERSION "${FLYTE_VERSION}" - -ARG FLYTE_TEST="release" -ENV FLYTE_TEST "${FLYTE_TEST}" - -# Update PATH variable -ENV PATH "/flyteorg/bin:${PATH}" - -# Declare volumes for k3s -VOLUME /var/lib/kubelet -VOLUME /var/lib/rancher/k3s -VOLUME /var/lib/cni -VOLUME /var/log - -# Expose Flyte ports -EXPOSE 30081 30082 30084 30088 - -ENTRYPOINT ["tini", "flyte-entrypoint.sh"] - - FROM docker:20.10.11-dind AS dind # Install dependencies @@ -96,7 +48,7 @@ RUN apk add --no-cache bash git make tini curl jq COPY --from=base_ /flyteorg/ /flyteorg/ # Copy entrypoints -COPY docker/sandbox/flyte-entrypoint-dind.sh /flyteorg/bin/flyte-entrypoint.sh +COPY docker/sandbox/flyte-entrypoint-default.sh /flyteorg/bin/flyte-entrypoint.sh ARG FLYTE_VERSION="latest" ENV FLYTE_VERSION "${FLYTE_VERSION}" diff --git a/docker/sandbox/flyte-entrypoint-default.sh b/docker/sandbox/flyte-entrypoint-default.sh index 9382b0b5c1..10d9f15615 100755 --- a/docker/sandbox/flyte-entrypoint-default.sh +++ b/docker/sandbox/flyte-entrypoint-default.sh @@ -7,10 +7,27 @@ cgroup-v2-hack.sh trap 'pkill -P $$' EXIT +monitor() { + while : ; do + for pid in $@ ; do + kill -0 $pid &> /dev/null || exit 1 + done + + sleep 1 + done +} + +# Start docker daemon +echo "Starting Docker daemon..." +dockerd &> /var/log/dockerd.log & +DOCKERD_PID=$! +timeout 600 sh -c "until docker info &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for dockerd to start"; exit 1 ) +echo "Done." + # Start k3s echo "Starting k3s cluster..." KUBERNETES_API_PORT=${KUBERNETES_API_PORT:-6443} -k3s server --no-deploy=traefik --no-deploy=servicelb --no-deploy=local-storage --no-deploy=metrics-server --https-listen-port=${KUBERNETES_API_PORT} &> /var/log/k3s.log & +k3s server --docker --no-deploy=traefik --no-deploy=servicelb --no-deploy=local-storage --no-deploy=metrics-server --https-listen-port=${KUBERNETES_API_PORT} &> /var/log/k3s.log & K3S_PID=$! timeout 600 sh -c "until k3s kubectl explain deployment &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for the Kubernetes cluster to start"; exit 1 ) echo "Done." @@ -23,7 +40,6 @@ fi # Deploy flyte echo "Deploying Flyte..." - version="" charts="/flyteorg/share/flyte" @@ -44,4 +60,10 @@ helm upgrade -n flyte --create-namespace flyte $charts --kubeconfig /etc/rancher wait-for-flyte.sh -wait ${K3S_PID} +# With flytectl sandbox --source flag, we mount the root volume to user source dir that will create helm & k8s cache specific directory. +# In Linux, These file belongs to root user that is different then current user +# In this case during fast serialization, Pyflyte will through error because of permission denied +rm -rf /root/.cache /root/.kube /root/.config + +# Monitor running processes. Exit when the first process exits. +monitor ${DOCKERD_PID} ${K3S_PID} diff --git a/docker/sandbox/flyte-entrypoint-dind.sh b/docker/sandbox/flyte-entrypoint-dind.sh deleted file mode 100755 index 10d9f15615..0000000000 --- a/docker/sandbox/flyte-entrypoint-dind.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/sh - -set -euo pipefail - -# Apply cgroup v2 hack -cgroup-v2-hack.sh - -trap 'pkill -P $$' EXIT - -monitor() { - while : ; do - for pid in $@ ; do - kill -0 $pid &> /dev/null || exit 1 - done - - sleep 1 - done -} - -# Start docker daemon -echo "Starting Docker daemon..." -dockerd &> /var/log/dockerd.log & -DOCKERD_PID=$! -timeout 600 sh -c "until docker info &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for dockerd to start"; exit 1 ) -echo "Done." - -# Start k3s -echo "Starting k3s cluster..." -KUBERNETES_API_PORT=${KUBERNETES_API_PORT:-6443} -k3s server --docker --no-deploy=traefik --no-deploy=servicelb --no-deploy=local-storage --no-deploy=metrics-server --https-listen-port=${KUBERNETES_API_PORT} &> /var/log/k3s.log & -K3S_PID=$! -timeout 600 sh -c "until k3s kubectl explain deployment &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for the Kubernetes cluster to start"; exit 1 ) -echo "Done." - -FLYTE_VERSION=${FLYTE_VERSION:-latest} -if [[ $FLYTE_VERSION = "latest" ]] -then - FLYTE_VERSION=$(curl --silent "https://api.github.com/repos/flyteorg/flyte/releases/latest" | jq -r .tag_name) -fi - -# Deploy flyte -echo "Deploying Flyte..." -version="" -charts="/flyteorg/share/flyte" - -if [[ $FLYTE_TEST = "release" ]] -then - helm repo add flyteorg https://flyteorg.github.io/flyte - helm fetch flyteorg/flyte --version=$FLYTE_VERSION - version="--version $FLYTE_VERSION" - charts="flyteorg/flyte" -fi - -if [[ $FLYTE_TEST = "local" ]] -then - helm dep update $charts -fi - -helm upgrade -n flyte --create-namespace flyte $charts --kubeconfig /etc/rancher/k3s/k3s.yaml --install $version - -wait-for-flyte.sh - -# With flytectl sandbox --source flag, we mount the root volume to user source dir that will create helm & k8s cache specific directory. -# In Linux, These file belongs to root user that is different then current user -# In this case during fast serialization, Pyflyte will through error because of permission denied -rm -rf /root/.cache /root/.kube /root/.config - -# Monitor running processes. Exit when the first process exits. -monitor ${DOCKERD_PID} ${K3S_PID}