From 1f4e1910273e7f1e09371b34781576066a375c00 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Thu, 12 Dec 2024 23:02:40 +0700 Subject: [PATCH 01/12] fix: Reduce multi container build action --- .github/workflows/docker-upload.yml | 99 ++++++++--------------------- 1 file changed, 28 insertions(+), 71 deletions(-) diff --git a/.github/workflows/docker-upload.yml b/.github/workflows/docker-upload.yml index 85448a02e..615afff2a 100644 --- a/.github/workflows/docker-upload.yml +++ b/.github/workflows/docker-upload.yml @@ -1,9 +1,9 @@ -name: Build and Publish Docker Image +name: Build nitro-eigenda Docker Image on: workflow_dispatch: jobs: - build-linux-amd64: + docker: runs-on: linux-2xl steps: @@ -23,9 +23,9 @@ jobs: - name: Cache Docker layers uses: actions/cache@v3 with: - path: /tmp/.buildx-cache-amd64 - key: linux-amd64-buildx-${{ hashFiles('Dockerfile') }} - restore-keys: linux-amd64-buildx- + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ hashFiles('Dockerfile') }} + restore-keys: ${{ runner.os }}-buildx- - name: Login to GitHub Container Registry uses: docker/login-action@v2 @@ -41,80 +41,37 @@ jobs: images: | ghcr.io/${{ github.repository_owner }}/nitro-eigenda tags: | - ${{ github.ref_name }}-linux-amd64 - latest-linux-amd64 - - - name: Build and push for linux/amd64 - uses: docker/build-push-action@v5 + type=ref,event=branch + type=ref,event=pr + type=ref,event=tag + type=ref,event=tag,suffix={{sha}} + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha,prefix=,format=short,enable=true + type=raw,value=latest,enable={{is_default_branch}} + type=match,pattern=^(v\d+\.\d+\.\d+-\w*)\..*$,value=$1,enable={{is_default_branch}} + + - name: "Build and push" + uses: docker/build-push-action@v6 with: - platform: linux/amd64 target: nitro-node-dev context: . + platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache-amd64 - cache-to: type=local,dest=/tmp/.buildx-cache-amd64-new,mode=max + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - name: Move cache run: | - rm -rf /tmp/.buildx-cache-amd64 - mv /tmp/.buildx-cache-amd64-new /tmp/.buildx-cache-amd64 - - build-linux-arm64: - runs-on: linux-2xl - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: 'recursive' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - install: true - - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache-arm64 - key: linux-arm64-buildx-${{ hashFiles('Dockerfile') }} - restore-keys: linux-arm64-buildx- - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: | - ghcr.io/${{ github.repository_owner }}/nitro-eigenda - tags: | - ${{ github.ref_name }}-linux-arm64 - latest-linux-arm64 + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache - - name: Build and push for linux/arm64 - uses: docker/build-push-action@v5 - with: - platform: linux/arm64/v8 - target: nitro-node-dev - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache-arm64 - cache-to: type=local,dest=/tmp/.buildx-cache-arm64-new,mode=max - - - name: Move cache + - name: Clear cache on failure + if: failure() run: | - rm -rf /tmp/.buildx-cache-arm64 - mv /tmp/.buildx-cache-arm64-new /tmp/.buildx-cache-arm64 + keys=(${{ runner.os }}-buildx- ${{ runner.os }}-buildx-${{ hashFiles('Dockerfile') }}) + for key in "${keys[@]}"; do + curl -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/actions/caches/$key" + done \ No newline at end of file From 0c949c02b80a5ddf73f17af60b372cf1042bea88 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Thu, 12 Dec 2024 23:03:58 +0700 Subject: [PATCH 02/12] fix: Reduce multi container build action --- .github/workflows/docker-upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-upload.yml b/.github/workflows/docker-upload.yml index 615afff2a..6bb46f702 100644 --- a/.github/workflows/docker-upload.yml +++ b/.github/workflows/docker-upload.yml @@ -1,4 +1,4 @@ -name: Build nitro-eigenda Docker Image +name: Build and Publish Docker Image on: workflow_dispatch: From 76354639624a96e3514a1e71ebdfaea46da157c8 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Thu, 12 Dec 2024 23:51:20 +0700 Subject: [PATCH 03/12] fix: Reduce multi container build action - test w/o multiplatform --- .github/workflows/docker-upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-upload.yml b/.github/workflows/docker-upload.yml index 6bb46f702..94ed4c95f 100644 --- a/.github/workflows/docker-upload.yml +++ b/.github/workflows/docker-upload.yml @@ -56,7 +56,7 @@ jobs: with: target: nitro-node-dev context: . - platforms: linux/amd64,linux/arm64 + # platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 5b9264bf4def784ec72641d8feae1a9e965f7ef5 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Fri, 13 Dec 2024 00:47:04 +0700 Subject: [PATCH 04/12] fix: Reduce multi container build action - multiplatform targets --- .github/workflows/docker-upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-upload.yml b/.github/workflows/docker-upload.yml index 94ed4c95f..6bb46f702 100644 --- a/.github/workflows/docker-upload.yml +++ b/.github/workflows/docker-upload.yml @@ -56,7 +56,7 @@ jobs: with: target: nitro-node-dev context: . - # platforms: linux/amd64,linux/arm64 + platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 24593195c5882248261fc86cd0a6e4cd2c6e0538 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Sat, 14 Dec 2024 12:43:08 +0700 Subject: [PATCH 05/12] fix: Reduce multi container build action - test w/o caching --- .github/workflows/docker-upload.yml | 38 ++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/docker-upload.yml b/.github/workflows/docker-upload.yml index 6bb46f702..24eb99276 100644 --- a/.github/workflows/docker-upload.yml +++ b/.github/workflows/docker-upload.yml @@ -20,12 +20,12 @@ jobs: with: install: true - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ hashFiles('Dockerfile') }} - restore-keys: ${{ runner.os }}-buildx- + # - name: Cache Docker layers + # uses: actions/cache@v3 + # with: + # path: /tmp/.buildx-cache + # key: ${{ runner.os }}-buildx-${{ hashFiles('Dockerfile') }} + # restore-keys: ${{ runner.os }}-buildx- - name: Login to GitHub Container Registry uses: docker/login-action@v2 @@ -60,18 +60,18 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + # cache-from: type=local,src=/tmp/.buildx-cache + # cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache + # - name: Move cache + # run: | + # rm -rf /tmp/.buildx-cache + # mv /tmp/.buildx-cache-new /tmp/.buildx-cache - - name: Clear cache on failure - if: failure() - run: | - keys=(${{ runner.os }}-buildx- ${{ runner.os }}-buildx-${{ hashFiles('Dockerfile') }}) - for key in "${keys[@]}"; do - curl -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/actions/caches/$key" - done \ No newline at end of file + # - name: Clear cache on failure + # if: failure() + # run: | + # keys=(${{ runner.os }}-buildx- ${{ runner.os }}-buildx-${{ hashFiles('Dockerfile') }}) + # for key in "${keys[@]}"; do + # curl -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/actions/caches/$key" + # done \ No newline at end of file From 343e59a69cc706bf276e717536a6cfaf964ed5e9 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Sat, 14 Dec 2024 16:49:46 +0700 Subject: [PATCH 06/12] fix: Reduce multi container build action - test as different jobs --- .github/workflows/docker-upload.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-upload.yml b/.github/workflows/docker-upload.yml index 24eb99276..0ad316b8b 100644 --- a/.github/workflows/docker-upload.yml +++ b/.github/workflows/docker-upload.yml @@ -51,12 +51,23 @@ jobs: type=raw,value=latest,enable={{is_default_branch}} type=match,pattern=^(v\d+\.\d+\.\d+-\w*)\..*$,value=$1,enable={{is_default_branch}} - - name: "Build and push" + - name: "Build and push amd64" uses: docker/build-push-action@v6 with: target: nitro-node-dev context: . - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + # cache-from: type=local,src=/tmp/.buildx-cache + # cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + - name: "Build and push arm64" + uses: docker/build-push-action@v6 + with: + target: nitro-node-dev + context: . + platforms: linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 9c59a3638bd7ecbfb43a934c8ebdbcfc3cdccdb8 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Sat, 14 Dec 2024 17:35:20 +0700 Subject: [PATCH 07/12] fix: Reduce multi container build action - test using newer build/push action --- .github/workflows/docker-upload.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-upload.yml b/.github/workflows/docker-upload.yml index 0ad316b8b..333c68ea7 100644 --- a/.github/workflows/docker-upload.yml +++ b/.github/workflows/docker-upload.yml @@ -52,7 +52,7 @@ jobs: type=match,pattern=^(v\d+\.\d+\.\d+-\w*)\..*$,value=$1,enable={{is_default_branch}} - name: "Build and push amd64" - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v6.10.0 with: target: nitro-node-dev context: . @@ -63,7 +63,7 @@ jobs: # cache-from: type=local,src=/tmp/.buildx-cache # cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - name: "Build and push arm64" - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v6.10.0 with: target: nitro-node-dev context: . From 73b4e85484e4c60c1d4ea76d47de286b8f0f5bce Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Sat, 14 Dec 2024 17:35:33 +0700 Subject: [PATCH 08/12] fix: Reduce multi container build action - test using newer build/push action --- .github/workflows/docker-upload.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-upload.yml b/.github/workflows/docker-upload.yml index 333c68ea7..2c759eab0 100644 --- a/.github/workflows/docker-upload.yml +++ b/.github/workflows/docker-upload.yml @@ -51,15 +51,15 @@ jobs: type=raw,value=latest,enable={{is_default_branch}} type=match,pattern=^(v\d+\.\d+\.\d+-\w*)\..*$,value=$1,enable={{is_default_branch}} - - name: "Build and push amd64" - uses: docker/build-push-action@v6.10.0 - with: - target: nitro-node-dev - context: . - platforms: linux/amd64 - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + # - name: "Build and push amd64" + # uses: docker/build-push-action@v6.10.0 + # with: + # target: nitro-node-dev + # context: . + # platforms: linux/amd64 + # push: true + # tags: ${{ steps.meta.outputs.tags }} + # labels: ${{ steps.meta.outputs.labels }} # cache-from: type=local,src=/tmp/.buildx-cache # cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - name: "Build and push arm64" From ae5664d5e84673b178572a47857e2040a0652dcc Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Sat, 14 Dec 2024 23:13:22 +0700 Subject: [PATCH 09/12] fix: Reduce multi container build action - specify different platform target --- .github/workflows/docker-upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-upload.yml b/.github/workflows/docker-upload.yml index 2c759eab0..2ae93408e 100644 --- a/.github/workflows/docker-upload.yml +++ b/.github/workflows/docker-upload.yml @@ -67,7 +67,7 @@ jobs: with: target: nitro-node-dev context: . - platforms: linux/arm64 + platforms: linux/arm64/v8 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 2a9138877f6d376fc252e16fb2f49a0258d6f2ab Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Sun, 15 Dec 2024 21:37:47 +0700 Subject: [PATCH 10/12] fix: Reduce multi container build action - test with tmate --- .github/workflows/docker-upload.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/docker-upload.yml b/.github/workflows/docker-upload.yml index 2ae93408e..704d37b04 100644 --- a/.github/workflows/docker-upload.yml +++ b/.github/workflows/docker-upload.yml @@ -62,6 +62,10 @@ jobs: # labels: ${{ steps.meta.outputs.labels }} # cache-from: type=local,src=/tmp/.buildx-cache # cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + - name: "Build and push arm64" uses: docker/build-push-action@v6.10.0 with: From 454f8cdd85e70cd52131d573c2fcb0ff4440d4b9 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Mon, 16 Dec 2024 22:21:22 +0700 Subject: [PATCH 11/12] fix: Reduce multi container build action - test arm64 builds using arm based machine --- .github/workflows/docker-upload.yml | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/.github/workflows/docker-upload.yml b/.github/workflows/docker-upload.yml index 704d37b04..0375b6294 100644 --- a/.github/workflows/docker-upload.yml +++ b/.github/workflows/docker-upload.yml @@ -4,7 +4,7 @@ on: jobs: docker: - runs-on: linux-2xl + runs-on: linux-xl-arm steps: - name: Checkout @@ -51,27 +51,12 @@ jobs: type=raw,value=latest,enable={{is_default_branch}} type=match,pattern=^(v\d+\.\d+\.\d+-\w*)\..*$,value=$1,enable={{is_default_branch}} - # - name: "Build and push amd64" - # uses: docker/build-push-action@v6.10.0 - # with: - # target: nitro-node-dev - # context: . - # platforms: linux/amd64 - # push: true - # tags: ${{ steps.meta.outputs.tags }} - # labels: ${{ steps.meta.outputs.labels }} - # cache-from: type=local,src=/tmp/.buildx-cache - # cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - - name: "Build and push arm64" uses: docker/build-push-action@v6.10.0 with: target: nitro-node-dev context: . - platforms: linux/arm64/v8 + platforms: linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 928c57d8543e26ecd1c16bdc32bb66bb47f21587 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Mon, 16 Dec 2024 23:15:20 +0700 Subject: [PATCH 12/12] fix: Reduce multi container build action - segment arm64 and amd64 builds --- .github/workflows/docker-upload.yml | 117 ++++++++++++++++++++++------ 1 file changed, 95 insertions(+), 22 deletions(-) diff --git a/.github/workflows/docker-upload.yml b/.github/workflows/docker-upload.yml index 0375b6294..6e785b793 100644 --- a/.github/workflows/docker-upload.yml +++ b/.github/workflows/docker-upload.yml @@ -3,7 +3,80 @@ on: workflow_dispatch: jobs: - docker: + docker-amd64: + runs-on: linux-2xl + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: 'recursive' + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + install: true + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ hashFiles('Dockerfile') }} + restore-keys: ${{ runner.os }}-buildx- + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ghcr.io/${{ github.repository_owner }}/nitro-eigenda + tags: | + type=ref,event=branch + type=ref,event=pr + type=ref,event=tag + type=ref,event=tag,suffix={{sha}} + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha,prefix=,format=short,enable=true + type=raw,value=latest,enable={{is_default_branch}} + type=match,pattern=^(v\d+\.\d+\.\d+-\w*)\..*$,value=$1,enable={{is_default_branch}} + + - name: "Build and push amd64" + uses: docker/build-push-action@v6.10.0 + with: + target: nitro-node-dev + context: . + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + + - name: Clear cache on failure + if: failure() + run: | + keys=(${{ runner.os }}-buildx- ${{ runner.os }}-buildx-${{ hashFiles('Dockerfile') }}) + for key in "${keys[@]}"; do + curl -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/actions/caches/$key" + done + + docker-arm64: runs-on: linux-xl-arm steps: @@ -20,12 +93,12 @@ jobs: with: install: true - # - name: Cache Docker layers - # uses: actions/cache@v3 - # with: - # path: /tmp/.buildx-cache - # key: ${{ runner.os }}-buildx-${{ hashFiles('Dockerfile') }} - # restore-keys: ${{ runner.os }}-buildx- + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ hashFiles('Dockerfile') }} + restore-keys: ${{ runner.os }}-buildx- - name: Login to GitHub Container Registry uses: docker/login-action@v2 @@ -60,18 +133,18 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - # cache-from: type=local,src=/tmp/.buildx-cache - # cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - - # - name: Move cache - # run: | - # rm -rf /tmp/.buildx-cache - # mv /tmp/.buildx-cache-new /tmp/.buildx-cache - - # - name: Clear cache on failure - # if: failure() - # run: | - # keys=(${{ runner.os }}-buildx- ${{ runner.os }}-buildx-${{ hashFiles('Dockerfile') }}) - # for key in "${keys[@]}"; do - # curl -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/actions/caches/$key" - # done \ No newline at end of file + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + + - name: Clear cache on failure + if: failure() + run: | + keys=(${{ runner.os }}-buildx- ${{ runner.os }}-buildx-${{ hashFiles('Dockerfile') }}) + for key in "${keys[@]}"; do + curl -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/actions/caches/$key" + done \ No newline at end of file