diff --git a/.github/workflows/sandbox.yml b/.github/workflows/sandbox.yml index 984dc697f6..ffc8a28478 100644 --- a/.github/workflows/sandbox.yml +++ b/.github/workflows/sandbox.yml @@ -3,11 +3,6 @@ name: Build & Push Sandbox Docker Image on: pull_request: paths: - # The flyte-binary image build is managed solely in this pipeline. Trigger - # the pipeline when there are any changes to the Dockerfile for this image. - # Ideally this image is built in a separate, standalone pipeline. - # TODO: Build flyte-binary image in a separate GHA pipeline - - Dockerfile - docker/** - charts/** - .github/workflows/sandbox.yml @@ -63,8 +58,6 @@ jobs: tags: ${{ steps.dind-names.outputs.tags }} build-args: "FLYTE_VERSION=${{ steps.set_version.outputs.flyte_version }}" file: docker/sandbox/Dockerfile - cache-from: type=gha - cache-to: type=gha,mode=max trigger-sandbox-lite-build: runs-on: ubuntu-latest @@ -122,152 +115,3 @@ jobs: tags: ${{ steps.dind-names.outputs.tags }} build-args: "FLYTE_VERSION=${{ steps.set_version.outputs.flyte_version }}" file: docker/sandbox-lite/Dockerfile - cache-from: type=gha - cache-to: type=gha,mode=max - - - trigger-single-binary-build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Setup Golang caches - uses: actions/cache@v3 - with: - path: | - /root/.cache/go-build - /root/go/pkg/mod - key: ${{ runner.os }}-golang-${{ hashFiles('go.sum') }} - restore-keys: | - ${{ runner.os }}-golang- - - name: Set flyte version to release - id: set_version - run: | - if [ ${{ github.event_name}} = "release" ]; then - echo ::set-output name=flyte_version::$(echo ${{ github.event.release.tag_name }}) - else - echo ::set-output name=flyte_version::latest - fi - - name: Prepare Image Names - id: image-names - uses: docker/metadata-action@v3 - with: - # list of Docker images to use as base name for tags - images: | - ghcr.io/${{ github.repository_owner }}/flyte-binary - tags: | - # set latest tag for default branch - type=raw,value=latest,enable={{is_default_branch}} - ${{ inputs.version }} - type=sha,format=long, - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - - name: Setup destination directories for image tarballs - run: | - mkdir -p docker/sandbox-bundled/images/tar/{arm64,amd64} - - name: Export ARM64 Image - uses: docker/build-push-action@v3 - env: - DOCKER_CLI_EXPERIMENTAL: enabled - with: - context: . - platforms: linux/arm64 - tags: flyte-binary:sandbox - build-args: "" - file: Dockerfile - cache-from: type=gha - cache-to: type=gha,mode=max - outputs: type=docker,dest=docker/sandbox-bundled/images/tar/arm64/flyte-binary.tar - - name: Export AMD64 Image - uses: docker/build-push-action@v3 - env: - DOCKER_CLI_EXPERIMENTAL: enabled - with: - context: . - platforms: linux/amd64 - tags: flyte-binary:sandbox - build-args: "" - file: Dockerfile - cache-from: type=gha - cache-to: type=gha,mode=max - outputs: type=docker,dest=docker/sandbox-bundled/images/tar/amd64/flyte-binary.tar - - name: Build and push Image - uses: docker/build-push-action@v3 - env: - DOCKER_CLI_EXPERIMENTAL: enabled - with: - context: . - platforms: linux/arm64, linux/amd64 - tags: ${{ steps.image-names.outputs.tags }} - build-args: "" - file: Dockerfile - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - - name: Upload Artifact GitHub Action - uses: actions/upload-artifact@v2 - with: - name: single-binary-image - path: docker/sandbox-bundled/images/tar - - trigger-sandbox-bundled-build: - runs-on: ubuntu-latest - needs: [trigger-single-binary-build] - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - uses: actions/download-artifact@v3 - with: - name: single-binary-image - path: docker/sandbox-bundled/images/tar - - name: Prepare Image Names - id: image-names - uses: docker/metadata-action@v3 - with: - # list of Docker images to use as base name for tags - images: | - ghcr.io/${{ github.repository_owner }}/flyte-sandbox-bundled - tags: | - # set latest tag for default branch - type=raw,value=latest,enable={{is_default_branch}} - ${{ inputs.version }} - type=sha,format=long, - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - with: - driver-opts: image=moby/buildkit:master - buildkitd-flags: "--allow-insecure-entitlement security.insecure" - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - - name: Build and push Image - uses: docker/build-push-action@v3 - env: - DOCKER_CLI_EXPERIMENTAL: enabled - with: - context: docker/sandbox-bundled - allow: "security.insecure" - platforms: linux/arm64, linux/amd64 - tags: ${{ steps.image-names.outputs.tags }} - cache-from: type=gha - cache-to: type=gha,mode=max - push: true diff --git a/.github/workflows/single-binary.yml b/.github/workflows/single-binary.yml new file mode 100644 index 0000000000..b992c7fa3d --- /dev/null +++ b/.github/workflows/single-binary.yml @@ -0,0 +1,168 @@ +name: Build & Push Flyte Single Binary Images + +on: + pull_request: + paths: + - .github/workflows/single-binary.yml + - charts/flyte-binary/** + - charts/flyte-sandbox/** + - cmd/** + - docker/sandbox-bundled/** + - Dockerfile + - go.* + push: + branches: + - master + paths: + - .github/workflows/single-binary.yml + - charts/flyte-binary/** + - charts/flyte-sandbox/** + - cmd/** + - docker/sandbox-bundled/** + - Dockerfile + - go.* + release: + types: [published] + +jobs: + trigger-single-binary-build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: "0" + - name: Setup Golang caches + uses: actions/cache@v3 + with: + path: | + /root/.cache/go-build + /root/go/pkg/mod + key: ${{ runner.os }}-golang-${{ hashFiles('go.sum') }} + restore-keys: | + ${{ runner.os }}-golang- + - name: Set flyte version to release + id: set_version + run: | + if [ ${{ github.event_name }} = "release" ]; then + echo "FLYTE_CONSOLE_VERSION=$(echo ${{ github.event.release.tag_name }})" >> $GITHUB_ENV + echo "FLYTE_VERSION=$(echo ${{ github.event.release.tag_name }})" >> $GITHUB_ENV + else + echo "FLYTE_CONSOLE_VERSION=latest" >> $GITHUB_ENV + echo "FLYTE_VERSION=${{ github.sha }}" >> $GITHUB_ENV + fi + - name: Prepare Image Names + id: image-names + uses: docker/metadata-action@v3 + with: + # list of Docker images to use as base name for tags + images: | + ghcr.io/${{ github.repository_owner }}/flyte-binary + tags: | + type=raw,value=latest,enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + type=sha,format=long + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: "${{ secrets.FLYTE_BOT_USERNAME }}" + password: "${{ secrets.FLYTE_BOT_PAT }}" + - name: Setup destination directories for image tarballs + run: | + mkdir -p docker/sandbox-bundled/images/tar/{arm64,amd64} + - name: Export ARM64 Image + uses: docker/build-push-action@v3 + env: + DOCKER_CLI_EXPERIMENTAL: enabled + with: + context: . + platforms: linux/arm64 + tags: flyte-binary:sandbox + build-args: | + FLYTE_CONSOLE_VERSION=${{ env.FLYTE_CONSOLE_VERSION }} + FLYTE_VERSION=${{ env.FLYTE_VERSION }} + file: Dockerfile + outputs: type=docker,dest=docker/sandbox-bundled/images/tar/arm64/flyte-binary.tar + - name: Export AMD64 Image + uses: docker/build-push-action@v3 + env: + DOCKER_CLI_EXPERIMENTAL: enabled + with: + context: . + platforms: linux/amd64 + tags: flyte-binary:sandbox + build-args: | + FLYTE_CONSOLE_VERSION=${{ env.FLYTE_CONSOLE_VERSION }} + FLYTE_VERSION=${{ env.FLYTE_VERSION }} + file: Dockerfile + outputs: type=docker,dest=docker/sandbox-bundled/images/tar/amd64/flyte-binary.tar + - name: Build and push Image + uses: docker/build-push-action@v3 + env: + DOCKER_CLI_EXPERIMENTAL: enabled + with: + context: . + platforms: linux/arm64, linux/amd64 + tags: ${{ steps.image-names.outputs.tags }} + build-args: | + FLYTE_CONSOLE_VERSION=${{ env.FLYTE_CONSOLE_VERSION }} + FLYTE_VERSION=${{ env.FLYTE_VERSION }} + file: Dockerfile + push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + - name: Upload Artifact GitHub Action + uses: actions/upload-artifact@v2 + with: + name: single-binary-image + path: docker/sandbox-bundled/images/tar + + trigger-sandbox-bundled-build: + runs-on: ubuntu-latest + needs: [trigger-single-binary-build] + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: "0" + - uses: actions/download-artifact@v3 + with: + name: single-binary-image + path: docker/sandbox-bundled/images/tar + - name: Prepare Image Names + id: image-names + uses: docker/metadata-action@v3 + with: + # list of Docker images to use as base name for tags + images: | + ghcr.io/${{ github.repository_owner }}/flyte-sandbox-bundled + tags: | + type=raw,value=latest,enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + type=sha,format=long, + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + driver-opts: image=moby/buildkit:master + buildkitd-flags: "--allow-insecure-entitlement security.insecure" + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: "${{ secrets.FLYTE_BOT_USERNAME }}" + password: "${{ secrets.FLYTE_BOT_PAT }}" + - name: Build and push Image + uses: docker/build-push-action@v3 + env: + DOCKER_CLI_EXPERIMENTAL: enabled + with: + context: docker/sandbox-bundled + allow: "security.insecure" + platforms: linux/arm64, linux/amd64 + tags: ${{ steps.image-names.outputs.tags }} + push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} diff --git a/Dockerfile b/Dockerfile index 243725c128..aae688f121 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,17 @@ -ARG FLYTE_VERSION="latest" +ARG FLYTE_CONSOLE_VERSION="latest" +FROM ghcr.io/flyteorg/flyteconsole-release:${FLYTE_CONSOLE_VERSION} AS flyteconsole -FROM ghcr.io/flyteorg/flyteconsole-release:$FLYTE_VERSION AS flyteconsole +FROM --platform=${BUILDPLATFORM} golang:1.19.1-bullseye AS flytebuilder -FROM golang:1.19.1-bullseye AS flytebuilder - -ARG FLYTE_VERSION="master" +ARG TARGETARCH +ENV GOARCH=${TARGETARCH} +ENV GOOS=linux WORKDIR /flyteorg/build -RUN git clone --depth=1 https://github.com/flyteorg/flyte.git ./flyte -b $FLYTE_VERSION -WORKDIR /flyteorg/build/flyte +COPY go.mod go.sum ./ RUN go mod download +COPY cmd cmd COPY --from=flyteconsole /app/dist cmd/single/dist RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/go/pkg/mod \ go build -tags console -v -o dist/flyte cmd/main.go @@ -18,8 +19,20 @@ RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/r FROM debian:bullseye-slim -ARG FLYTE_VERSION="master" +ARG FLYTE_VERSION="" + +ENV DEBCONF_NONINTERACTIVE_SEEN true +ENV DEBIAN_FRONTEND noninteractive ENV FLYTE_VERSION=${FLYTE_VERSION} -COPY --from=flytebuilder /flyteorg/build/flyte/dist/flyte /usr/local/bin/ -ENTRYPOINT [ "flyte" ] +# Install core packages +RUN apt-get update && apt-get install --no-install-recommends --yes \ + ca-certificates \ + tini \ + && rm -rf /var/lib/apt/lists/* + +# Copy compiled executable into image +COPY --from=flytebuilder /flyteorg/build/dist/flyte /usr/local/bin/ + +# Set entrypoint +ENTRYPOINT [ "/usr/bin/tini", "-g", "--", "/usr/local/bin/flyte" ] diff --git a/docker/sandbox-lite/Dockerfile b/docker/sandbox-lite/Dockerfile index 8b8faeedc5..fcac3259b8 100644 --- a/docker/sandbox-lite/Dockerfile +++ b/docker/sandbox-lite/Dockerfile @@ -4,7 +4,11 @@ ARG FLYTE_VERSION="latest" FROM ghcr.io/flyteorg/flyteconsole-release:${FLYTE_VERSION} AS flyteconsole -FROM golang:1.18.0-alpine3.15 AS go_builder +FROM --platform=${BUILDPLATFORM} golang:1.18.0-alpine3.15 AS go_builder + +ARG TARGETARCH +ENV GOARCH=${TARGETARCH} +ENV GOOS=linux # Install dependencies RUN apk add --no-cache build-base