From 82c3e3f7e23d83f3644bb9a41f4461273ed980b9 Mon Sep 17 00:00:00 2001 From: Michael FIG Date: Tue, 3 Sep 2024 23:19:03 -0600 Subject: [PATCH] ci(docker): use Depot multiarch, publish use-upgrade-next --- .github/workflows/docker.yml | 275 ++++++++++++++++-------------- .github/workflows/integration.yml | 2 +- depot.json | 1 + 3 files changed, 151 insertions(+), 127 deletions(-) create mode 100644 depot.json diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 21054c163d30..320b54f3d815 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -12,6 +12,7 @@ on: workflow_dispatch: env: REGISTRY: ghcr.io + DOCKER_PLATFORMS: linux/amd64,linux/arm64/v8 jobs: snapshot: @@ -33,23 +34,35 @@ jobs: SNAPSHOT_TAG="${TIMESTAMP}-${SHORT_SHA}" echo "tag=$SNAPSHOT_TAG" >> $GITHUB_OUTPUT - docker-parallel-build: - needs: snapshot - runs-on: ubuntu-latest + docker-sdk: permissions: + # allow issuing OIDC tokens for this workflow run + id-token: write + # allow at least reading the repo contents, add other permissions if necessary contents: read + # to push the resulting images packages: write - strategy: - matrix: - platform: - - linux/amd64 - - linux/arm64/v8 + needs: snapshot + runs-on: 'depot-ubuntu-22.04-16' # ubuntu-latest + outputs: + tag: '${{ steps.docker-tags.outputs.tags }}' + tags: '${{ steps.docker-tags.outputs.tags }} ${{ needs.snapshot.outputs.tag }}' steps: - uses: actions/checkout@v4 + - uses: depot/setup-action@v1 + with: + oidc: true # to set DEPOT_TOKEN for later steps + - run: depot configure-docker + - name: Log in to the Container registry + uses: docker/login-action@v3 + # see https://docs.github.com/en/actions/publishing-packages/publishing-docker-images + with: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: ${{ env.REGISTRY }} - name: Save BUILD_TAG run: | - ARCH=$(echo '${{ matrix.platform }}' | tr / _) - echo "BUILD_TAG=${{ needs.snapshot.outputs.tag }}-$ARCH" >> $GITHUB_ENV + echo "BUILD_TAG=${{ needs.snapshot.outputs.tag }}" >> $GITHUB_ENV - name: Save GIT_REVISION run: echo "GIT_REVISION=$(git rev-parse HEAD)" >> $GITHUB_ENV - name: Save GIT_COMMIT @@ -57,85 +70,14 @@ jobs: - name: Save commit hash, url of submodules to environment run: | node packages/xsnap/src/build.js --show-env >> $GITHUB_ENV - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - buildkitd-flags: --debug - - name: Login to Docker Registry - uses: docker/login-action@v2 - with: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: ${{ env.REGISTRY }} - name: Build and Push ssh-node - uses: docker/build-push-action@v4 + uses: depot/build-push-action@v1 with: file: packages/deployment/Dockerfile.ssh-node context: packages/deployment/docker - platforms: ${{ matrix.platform }} + platforms: ${{ env.DOCKER_PLATFORMS }} push: true tags: '${{ env.REGISTRY }}/agoric/ssh-node:${{ env.BUILD_TAG }}' - - name: Build and Push sdk - uses: docker/build-push-action@v4 - with: - file: packages/deployment/Dockerfile.sdk - context: ./ - platforms: ${{ matrix.platform }} - push: true - tags: '${{ env.REGISTRY }}/agoric/agoric-sdk:${{ env.BUILD_TAG }}' - # When changing/adding entries here, make sure to search the whole - # project for `@@AGORIC_DOCKER_SUBMODULES@@` - build-args: | - GIT_COMMIT=${{env.GIT_COMMIT}} - MODDABLE_COMMIT_HASH=${{env.MODDABLE_COMMIT_HASH}} - MODDABLE_URL=${{env.MODDABLE_URL}} - XSNAP_NATIVE_COMMIT_HASH=${{env.XSNAP_NATIVE_COMMIT_HASH}} - XSNAP_NATIVE_URL=${{env.XSNAP_NATIVE_URL}} - GIT_REVISION=${{env.GIT_REVISION}} - - name: Build and Push setup - uses: docker/build-push-action@v4 - with: - file: packages/deployment/Dockerfile - context: packages/deployment - platforms: ${{ matrix.platform }} - tags: '${{ env.REGISTRY }}/agoric/cosmic-swingset-setup:${{ env.BUILD_TAG }}' - push: true - build-args: | - TAG=${{ env.BUILD_TAG }} - - name: notify on failure - if: failure() - uses: ./.github/actions/notify-status - with: - webhook: ${{ secrets.SLACK_WEBHOOK_URL }} - from: ${{ secrets.NOTIFY_EMAIL_FROM }} - to: ${{ secrets.NOTIFY_EMAIL_TO }} - password: ${{ secrets.NOTIFY_EMAIL_PASSWORD }} - - # Publish the build's multiarch images to Docker Registry. - docker-sdk: - needs: [docker-parallel-build, snapshot] - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - outputs: - tag: '${{ steps.docker-tags.outputs.tags }}' - tags: '${{ steps.docker-tags.outputs.tags }} ${{ needs.snapshot.outputs.tag }}' - steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - buildkitd-flags: --debug - - name: Login to Docker Registry - uses: docker/login-action@v2 - with: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: ${{ env.REGISTRY }} - name: Compute tags id: docker-tags run: | @@ -168,67 +110,146 @@ jobs: DOCKER_TAGS="latest $SDK_TAG" ;; esac + DOCKER_TAGS="$DOCKER_TAGS $BUILD_TAG" echo "tags=$DOCKER_TAGS" >> $GITHUB_OUTPUT - - - name: Push SDK multiarch + - name: Prefix tags + id: prefix-tags run: | - set -ex - for IMAGE in agoric/agoric-sdk agoric/ssh-node agoric/cosmic-swingset-setup; do - for TAG in ${{ steps.docker-tags.outputs.tags }} ${{ needs.snapshot.outputs.tag }}; do - sources= - for ARCH in linux/amd64 linux/arm64/v8; do - uarch=$(echo "$ARCH" | tr / _) - BUILD_TAG="${{ needs.snapshot.outputs.tag }}-$uarch" - sources="$sources $REGISTRY/$IMAGE:$BUILD_TAG" - done - docker buildx imagetools create --tag "$REGISTRY/$IMAGE:$TAG"$sources - done + IMAGE="$REGISTRY/agoric/agoric-sdk" + for TAG in ${{ steps.docker-tags.outputs.tags }}; do + PREFIXED="$PREFIXED$sep$IMAGE:$TAG" + sep=, done + echo "tags=$PREFIXED" >> $GITHUB_OUTPUT + - name: Build and Push sdk + uses: depot/build-push-action@v1 + with: + file: packages/deployment/Dockerfile.sdk + context: ./ + platforms: ${{ env.DOCKER_PLATFORMS }} + push: true + tags: '${{ steps.prefix-tags.outputs.tags }}' + # When changing/adding entries here, make sure to search the whole + # project for `@@AGORIC_DOCKER_SUBMODULES@@` + build-args: | + GIT_COMMIT=${{env.GIT_COMMIT}} + MODDABLE_COMMIT_HASH=${{env.MODDABLE_COMMIT_HASH}} + MODDABLE_URL=${{env.MODDABLE_URL}} + XSNAP_NATIVE_COMMIT_HASH=${{env.XSNAP_NATIVE_COMMIT_HASH}} + XSNAP_NATIVE_URL=${{env.XSNAP_NATIVE_URL}} + GIT_REVISION=${{env.GIT_REVISION}} + - name: Build and Push setup + uses: depot/build-push-action@v1 + with: + file: packages/deployment/Dockerfile + context: packages/deployment + platforms: ${{ env.DOCKER_PLATFORMS }} + tags: '${{ env.REGISTRY }}/agoric/cosmic-swingset-setup:${{ env.BUILD_TAG }}' + push: true + build-args: | + TAG=${{ env.BUILD_TAG }} + - name: notify on failure + if: failure() + uses: ./.github/actions/notify-status + with: + webhook: ${{ secrets.SLACK_WEBHOOK_URL }} + from: ${{ secrets.NOTIFY_EMAIL_FROM }} + to: ${{ secrets.NOTIFY_EMAIL_TO }} + password: ${{ secrets.NOTIFY_EMAIL_PASSWORD }} - # This is currently needed for the relayer integration test framework. - # It just runs agoric/agoric-sdk with a "single-node" argument. - docker-ibc-alpha: + docker-build-upgrade-next: needs: [docker-sdk, snapshot] - runs-on: ubuntu-latest + runs-on: 'depot-ubuntu-22.04-16' # ubuntu-latest permissions: + # allow issuing OIDC tokens for this workflow run + id-token: write + # allow at least reading the repo contents, add other permissions if necessary contents: read + # to push the resulting images packages: write - if: ${{ needs.docker-sdk.outputs.tag }} != dev + outputs: + build-id: '${{ steps.bake-upgrade-next.outputs.build-id }}' + tags: '${{ steps.prefix-tags.outputs.tags }}' steps: - uses: actions/checkout@v4 + - uses: ./.github/actions/restore-node + with: + # 18.19 causes this problem running tsx: + # The --loader flag was deprecated in Node v20.6.0 + node-version: 18.18 + # Since we can't build core eval submissions for the SDK using a newer endo, + # simply ignore any endo branch integration (this means we don't have full coverage) + ignore-endo-branch: 'true' + + - name: Install deps + run: | + # Enable corepack for packageManager config + corepack enable || sudo corepack enable + yarn install + working-directory: a3p-integration + - name: Prepare Docker config + run: | + # prepare files for bake-action + yarn build:submissions + yarn synthetic-chain prepare-build + working-directory: a3p-integration + - name: Save SDK_TAG run: echo "SDK_TAG=${{ needs.snapshot.outputs.tag }}" >> $GITHUB_ENV + + - name: Override the docker-bake.json variables + run: | + REPOSITORY_COLON="$REGISTRY/agoric/agoric-sdk:a3p-" + jq -n --arg REPOSITORY_COLON "$REPOSITORY_COLON" \ + --arg PLATFORMS "$DOCKER_PLATFORMS" \ + '{"variable": { + "PLATFORMS": { "default": ($PLATFORMS | split(",")) }, + "REPOSITORY_COLON": { "default": $REPOSITORY_COLON }, + }}' > ./docker-bake.override.json + working-directory: a3p-integration + - name: Prefix tags id: prefix-tags run: | IMAGE="$REGISTRY/agoric/agoric-sdk" - for TAG in ibc-alpha; do - PREFIXED="$PREFIXED$sep$IMAGE:$TAG" + for TAG in ${{ needs.docker-sdk.outputs.tags }}; do + PREFIXED="$PREFIXED$sep$IMAGE:a3p-use-upgrade-next-$TAG" sep=, done echo "tags=$PREFIXED" >> $GITHUB_OUTPUT - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + - uses: depot/setup-action@v1 with: - buildkitd-flags: --debug + oidc: true # to set DEPOT_TOKEN for later steps + - run: depot configure-docker - name: Login to Docker Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} registry: ${{ env.REGISTRY }} - - name: Build and Push - uses: docker/build-push-action@v4 + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 with: - file: packages/deployment/Dockerfile.ibc-alpha - context: packages/deployment/docker - platforms: linux/amd64,linux/arm64/v8 + images: ${{ env.REGISTRY }}/agoric/agoric-sdk + tags: ${{ steps.prefix-tags.outputs.tags }} + + - name: Publish "use-upgrade-next" image + uses: depot/bake-action@v1 + id: bake-upgrade-next + with: + files: | + docker-bake.json + docker-bake.override.json + docker-bake.hcl + ${{ steps.meta.outputs.bake-file }} + set: '*.args.SDK_TAG=${{ env.SDK_TAG }}' + targets: use-upgrade-next push: true - tags: '${{ steps.prefix-tags.outputs.tags }}' - build-args: | - SDK_TAG=${{env.SDK_TAG}} + workdir: ./a3p-integration + + - name: clean up repo + run: git clean -f && git restore . - name: notify on failure if: failure() uses: ./.github/actions/notify-status @@ -240,9 +261,13 @@ jobs: docker-solo: needs: [docker-sdk, snapshot] - runs-on: ubuntu-latest + runs-on: 'depot-ubuntu-22.04-16' # ubuntu-latest permissions: + # allow issuing OIDC tokens for this workflow run + id-token: write + # allow at least reading the repo contents, add other permissions if necessary contents: read + # to push the resulting images packages: write steps: - uses: actions/checkout@v4 @@ -257,24 +282,22 @@ jobs: sep=, done echo "tags=$PREFIXED" >> $GITHUB_OUTPUT - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + - uses: depot/setup-action@v1 with: - buildkitd-flags: --debug + oidc: true # to set DEPOT_TOKEN for later steps + - run: depot configure-docker - name: Login to Docker Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} registry: ${{ env.REGISTRY }} - name: Build and Push - uses: docker/build-push-action@v4 + uses: depot/build-push-action@v1 with: file: packages/solo/Dockerfile context: packages/solo - platforms: linux/amd64,linux/arm64/v8 + platforms: ${{ env.DOCKER_PLATFORMS }} push: true tags: '${{ steps.prefix-tags.outputs.tags }}' build-args: | diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 2590ef712a36..a01d25ebad19 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -248,7 +248,7 @@ jobs: with: node-version: '18.x' # Rebuilding the SDK image with resolved endo packages is not currently supported - # and since we can't build core eval submissions form the SDK using a newer endo, + # and since we can't build core eval submissions from the SDK using a newer endo, # simply ignore any endo branch integration (this means we don't have full coverage) ignore-endo-branch: 'true' id: restore-node diff --git a/depot.json b/depot.json new file mode 100644 index 000000000000..6687457c3347 --- /dev/null +++ b/depot.json @@ -0,0 +1 @@ +{"id":"0gtvlr9g8j"}