From 38d66b1f8929d2381ae43fe388343c3e7e9a630e Mon Sep 17 00:00:00 2001 From: Aleksandr Stepanov Date: Mon, 2 Oct 2023 14:56:38 +0300 Subject: [PATCH 1/2] ci: Migration from ci-runner to matterlabs-ci-runner (#133) --- .github/workflows/build-external-node-docker.yml | 2 +- .github/workflows/build-gar-reusable.yml | 2 +- .github/workflows/build-local-node-docker.yml | 2 +- .github/workflows/build-prover-fri-gpu-gar.yml | 2 +- .github/workflows/coverage.yml | 2 +- .github/workflows/vm-perf-comparison.yml | 2 +- .github/workflows/vm-perf-to-prometheus.yml | 2 +- .github/workflows/zk-environment-cuda-12-0.publish.yml | 2 +- .github/workflows/zk-environment.publish.yml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-external-node-docker.yml b/.github/workflows/build-external-node-docker.yml index 31ec1e81c093..cccbc84c6c9b 100644 --- a/.github/workflows/build-external-node-docker.yml +++ b/.github/workflows/build-external-node-docker.yml @@ -11,7 +11,7 @@ on: jobs: build-images: name: External Node - Build and Push Docker Image - runs-on: [self-hosted, ci-runner] + runs-on: [matterlabs-ci-runner] steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 with: diff --git a/.github/workflows/build-gar-reusable.yml b/.github/workflows/build-gar-reusable.yml index 745ff6666e13..9a14508fdc7a 100644 --- a/.github/workflows/build-gar-reusable.yml +++ b/.github/workflows/build-gar-reusable.yml @@ -20,7 +20,7 @@ on: jobs: build-gar-prover: name: Build GAR prover - runs-on: [self-hosted, ci-runner] + runs-on: [matterlabs-ci-runner] strategy: fail-fast: false matrix: diff --git a/.github/workflows/build-local-node-docker.yml b/.github/workflows/build-local-node-docker.yml index 5f4cfa4547eb..9880361206c1 100644 --- a/.github/workflows/build-local-node-docker.yml +++ b/.github/workflows/build-local-node-docker.yml @@ -11,7 +11,7 @@ on: jobs: build-images: name: Local Node - Build and Push Docker Image - runs-on: [self-hosted, ci-runner] + runs-on: [matterlabs-ci-runner] steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 with: diff --git a/.github/workflows/build-prover-fri-gpu-gar.yml b/.github/workflows/build-prover-fri-gpu-gar.yml index 61304a5eb2c9..9643d9433188 100644 --- a/.github/workflows/build-prover-fri-gpu-gar.yml +++ b/.github/workflows/build-prover-fri-gpu-gar.yml @@ -15,7 +15,7 @@ on: jobs: build-gar-prover-fri-gpu: name: Build prover FRI GPU GAR - runs-on: [self-hosted, ci-runner] + runs-on: [matterlabs-ci-runner] steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 with: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 5064bfe22722..e7ed89eb760a 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -7,7 +7,7 @@ on: jobs: generate: - runs-on: [self-hosted, ci-runner] + runs-on: [matterlabs-ci-runner] steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 diff --git a/.github/workflows/vm-perf-comparison.yml b/.github/workflows/vm-perf-comparison.yml index 3078c9bfa8a1..1e5b65a35989 100644 --- a/.github/workflows/vm-perf-comparison.yml +++ b/.github/workflows/vm-perf-comparison.yml @@ -6,7 +6,7 @@ on: jobs: vm-benchmarks: name: Run VM benchmarks - runs-on: [self-hosted, ci-runner] + runs-on: [matterlabs-ci-runner] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/vm-perf-to-prometheus.yml b/.github/workflows/vm-perf-to-prometheus.yml index c5fe27632736..d2a6594ffca2 100644 --- a/.github/workflows/vm-perf-to-prometheus.yml +++ b/.github/workflows/vm-perf-to-prometheus.yml @@ -12,7 +12,7 @@ concurrency: vm-benchmarks jobs: vm-benchmarks: name: Run VM benchmarks - runs-on: [self-hosted, ci-runner] + runs-on: [matterlabs-ci-runner] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/zk-environment-cuda-12-0.publish.yml b/.github/workflows/zk-environment-cuda-12-0.publish.yml index f6471582bdf0..7018a61117ff 100644 --- a/.github/workflows/zk-environment-cuda-12-0.publish.yml +++ b/.github/workflows/zk-environment-cuda-12-0.publish.yml @@ -16,7 +16,7 @@ jobs: push_to_registry: if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' name: Push Docker image to Docker Hub - runs-on: [self-hosted, ci-runner] + runs-on: [matterlabs-ci-runner] steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 with: diff --git a/.github/workflows/zk-environment.publish.yml b/.github/workflows/zk-environment.publish.yml index 561da6190210..6070d37c7071 100644 --- a/.github/workflows/zk-environment.publish.yml +++ b/.github/workflows/zk-environment.publish.yml @@ -16,7 +16,7 @@ jobs: push_to_registry: if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' name: Push Docker image to Docker Hub - runs-on: [self-hosted, ci-runner] + runs-on: [matterlabs-ci-runner] steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 with: From 8df11278ca76000d842fc0f73f5233dfc85ef77e Mon Sep 17 00:00:00 2001 From: Maksym Date: Mon, 2 Oct 2023 15:48:39 +0300 Subject: [PATCH 2/2] ci: refactor zk env workflow (#135) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # What ❔ ## Why ❔ Merge jobs which has something to do with zk-environment to one workflow ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Tests for the changes have been added / updated. - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. --- .github/workflows/ci.yml | 5 + .../zk-environment-cuda-12-0.publish.yml | 50 -------- .github/workflows/zk-environment.publish.yml | 108 +++++++++++++++++- 3 files changed, 108 insertions(+), 55 deletions(-) delete mode 100644 .github/workflows/zk-environment-cuda-12-0.publish.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 091941a66aa9..2de28c23d154 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,11 @@ name: CI on: pull_request: + paths-ignore: + - ".github/workflows/zk-environment.publish.yml" + - "docker/zk-environment/Dockerfile" + - "docker/zk-environment-cuda-12-0/Dockerfile" + - "docker/zk-rust-nightly-environment/Dockerfile" merge_group: push: branches: diff --git a/.github/workflows/zk-environment-cuda-12-0.publish.yml b/.github/workflows/zk-environment-cuda-12-0.publish.yml deleted file mode 100644 index 7018a61117ff..000000000000 --- a/.github/workflows/zk-environment-cuda-12-0.publish.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: publish zk-environment with cuda 12.0 docker image - -on: - pull_request: - branches: - - main - types: [closed] - paths: - - "docker/zk-environment-cuda-12-0/Dockerfile" - - ".github/workflows/zk-environment-cuda-12-0.publish.yml" - workflow_dispatch: - branches: - - "main" - -jobs: - push_to_registry: - if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' - name: Push Docker image to Docker Hub - runs-on: [matterlabs-ci-runner] - steps: - - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 - with: - submodules: "recursive" - - - name: Login to us-central1 GAR - run: | - gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev - - - name: Log in to Docker Hub - uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Build and push - uses: docker/build-push-action@v4 - with: - context: . - file: docker/zk-environment-cuda-12-0/Dockerfile - push: true - target: nvidia-tools - tags: | - us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/zk-environment-cuda-12-0:latest - matterlabs/zk-environment:cuda-12-0-latest diff --git a/.github/workflows/zk-environment.publish.yml b/.github/workflows/zk-environment.publish.yml index 6070d37c7071..2fb5c1d88135 100644 --- a/.github/workflows/zk-environment.publish.yml +++ b/.github/workflows/zk-environment.publish.yml @@ -1,4 +1,4 @@ -name: publish zk-environment docker image +name: publish zk-environment docker images on: pull_request: @@ -8,15 +8,47 @@ on: paths: - "docker/zk-environment/Dockerfile" - ".github/workflows/zk-environment.publish.yml" + - "docker/zk-environment-cuda-12-0/Dockerfile" + - ".github/workflows/zk-environment-cuda-12-0.publish.yml" + - "docker/zk-rust-nightly-environment/Dockerfile" + - ".github/workflows/rust-nightly-environment.publish.yml" workflow_dispatch: branches: - "main" jobs: - push_to_registry: - if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' - name: Push Docker image to Docker Hub - runs-on: [matterlabs-ci-runner] + changed_files: + name: Changed files + outputs: + zk_environment: ${{ steps.changed-files-yaml.outputs.zk_env_any_changed }} + zk_environment_cuda_12: ${{ steps.changed-files-yaml.outputs.zk_env_cuda_12_any_changed }} + rust_nightly: ${{ steps.changed-files-yaml.outputs.rust_nightly_any_changed }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + with: + submodules: "recursive" + + - name: Get changed files + id: changed-files-yaml + uses: tj-actions/changed-files@v39 + with: + files_yaml: | + zk_env: + - docker/zk-environment/Dockerfile + - .github/workflows/zk-environment.publish.yml + zk_env_cuda_12: + - docker/zk-environment-cuda-12-0/Dockerfile + - .github/workflows/zk-environment-cuda-12-0.publish.yml + rust_nightly: + - docker/zk-rust-nightly-environment/Dockerfile + - .github/workflows/rust-nightly-environment.publish.yml + + zk_environment: + if: needs.changed_files.outputs.zk_environment == 'true' && github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' + name: Push zk-environment docker image to Docker Hub + runs-on: ubuntu-latest + needs: changed_files steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 with: @@ -51,3 +83,69 @@ jobs: tags: "matterlabs/zk-environment:latest2.0" file: docker/zk-environment/Dockerfile no-cache: true + + rust_nightly: + if: needs.changed_files.outputs.rust_nightly == 'true' && github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' + name: Push rust nightly docker image to Docker Hub + needs: changed_files + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + with: + submodules: "recursive" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v2 + + - name: Log in to Docker Hub + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0 + with: + context: . + push: true + tags: "matterlabs/zksync_rust:nightly" + file: docker/zk-rust-nightly-environment/Dockerfile + no-cache: true + + + zk_environment_cuda_12: + if: needs.changed_files.outputs.zk_environment_cuda_12 == 'true' github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' + name: Push zk-environment cuda 12 docker image to Docker Hub + runs-on: [matterlabs-ci-runner] + needs: changed_files + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + with: + submodules: "recursive" + + - name: Login to us-central1 GAR + run: | + gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev + + - name: Log in to Docker Hub + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and push + uses: docker/build-push-action@v4 + with: + context: . + file: docker/zk-environment-cuda-12-0/Dockerfile + push: true + target: nvidia-tools + tags: | + us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/zk-environment-cuda-12-0:latest + matterlabs/zk-environment:cuda-12-0-latest