diff --git a/.github/workflows/build-core-template.yml b/.github/workflows/build-core-template.yml new file mode 100644 index 00000000000..4857ecb8e61 --- /dev/null +++ b/.github/workflows/build-core-template.yml @@ -0,0 +1,87 @@ +name: Build Core images +on: + workflow_call: + secrets: + DOCKERHUB_USER: + description: "DOCKERHUB_USER" + required: true + DOCKERHUB_TOKEN: + description: "DOCKERHUB_TOKEN" + required: true + inputs: + image_tag: + description: "Tag of a built image to deploy" + type: string + required: true + image_tag_suffix: + description: "Optional suffix to override tag name generation" + type: string + required: false + action: + description: "Action with docker image" + type: string + default: "push" + required: false + +jobs: + build-images: + name: Build and Push Docker Images + env: + image_tag: ${{ inputs.image_tag }} + IMAGE_TAG_SUFFIX: ${{ inputs.image_tag_suffix }} + runs-on: [matterlabs-ci-runner] + strategy: + matrix: + component: + - server-v2 + - external-node + - contract-verifier + - cross-external-nodes-checker + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + with: + submodules: "recursive" + + - name: setup-env + run: | + echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV + echo CI=1 >> $GITHUB_ENV + echo $(pwd)/bin >> $GITHUB_PATH + echo CI=1 >> .env + echo IN_DOCKER=1 >> .env + + - name: start-services + run: | + echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env + docker-compose -f docker-compose-runner.yml up -d zk geth postgres + ci_run sccache --start-server + + - name: init + run: | + ci_run git config --global --add safe.directory /usr/src/zksync + ci_run git config --global --add safe.directory /usr/src/zksync/sdk/binaryen + ci_run git config --global --add safe.directory /usr/src/zksync/etc/system-contracts + ci_run git config --global --add safe.directory /usr/src/zksync/contracts + ci_run zk + ci_run zk clean --all + ci_run zk run yarn + ci_run cp etc/tokens/{test,localhost}.json + ci_run zk compiler all + ci_run zk contract build + ci_run zk f yarn run l2-contracts build + ci_run curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^26.key + + - name: update-images + env: + DOCKER_ACTION: ${{ inputs.action }} + COMPONENT: ${{ matrix.component }} + run: | + ci_run docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }} + ci_run gcloud auth configure-docker us-docker.pkg.dev,asia-docker.pkg.dev -q + ci_run zk docker $DOCKER_ACTION $COMPONENT -- --public + + - name: Show sccache stats + if: always() + run: | + ci_run sccache --show-stats + ci_run cat /tmp/sccache_log.txt diff --git a/.github/workflows/build-prover-template.yml b/.github/workflows/build-prover-template.yml new file mode 100644 index 00000000000..f084bb3c382 --- /dev/null +++ b/.github/workflows/build-prover-template.yml @@ -0,0 +1,142 @@ +name: Build Prover images +on: + workflow_call: + secrets: + DOCKERHUB_USER: + description: "DOCKERHUB_USER" + required: true + DOCKERHUB_TOKEN: + description: "DOCKERHUB_TOKEN" + required: true + inputs: + ERA_BELLMAN_CUDA_RELEASE: + description: "ERA_BELLMAN_CUDA_RELEASE" + type: string + required: true + image_tag: + description: "Tag of a built image to deploy" + type: string + required: true + image_tag_suffix: + description: "Optional suffix to override tag name generation" + type: string + required: false + action: + description: "Action with docker image" + type: string + default: "push" + required: false + +jobs: + era-bellman-cuda: + name: Get era-bellman-cuda release URLs. + runs-on: [ubuntu-latest] + outputs: + source: ${{ steps.release.outputs.source }} + binary: ${{ steps.release.outputs.binary }} + steps: + - id: release + run: | + release=($(curl -v --silent https://api.github.com/repos/matter-labs/era-bellman-cuda/releases/tags/${{ inputs.ERA_BELLMAN_CUDA_RELEASE }} | jq -r ".name, .tarball_url, .assets[0].browser_download_url")) + echo "source=${release[1]}" >> "$GITHUB_OUTPUT" + echo "binary=${release[2]}" >> "$GITHUB_OUTPUT" + + build-images: + name: Build and Push Docker Images + env: + image_tag: ${{ inputs.image_tag }} + IMAGE_TAG_SUFFIX: ${{ inputs.image_tag_suffix }} + runs-on: [matterlabs-ci-runner] + needs: [era-bellman-cuda] + strategy: + matrix: + component: + - witness-generator + - prover-v2 + - circuit-synthesizer + - prover-fri + - prover-gpu-fri + - witness-vector-generator + - prover-fri-gateway + - proof-fri-compressor + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + with: + submodules: "recursive" + + - name: Prepare bellman-cuda directory + run: etc/scripts/prepare_era_bellman_cuda.sh ${{ needs.era-bellman-cuda.outputs.source }} ${{ needs.era-bellman-cuda.outputs.binary }} + + - name: setup-env + run: | + echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV + echo CI=1 >> $GITHUB_ENV + echo $(pwd)/bin >> $GITHUB_PATH + echo CI=1 >> .env + echo IN_DOCKER=1 >> .env + + - name: start-services + run: | + echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env + docker-compose -f docker-compose-runner.yml up -d zk geth postgres + ci_run sccache --start-server + + - name: init + run: | + ci_run git config --global --add safe.directory /usr/src/zksync + ci_run git config --global --add safe.directory /usr/src/zksync/sdk/binaryen + ci_run git config --global --add safe.directory /usr/src/zksync/etc/system-contracts + ci_run git config --global --add safe.directory /usr/src/zksync/contracts + ci_run zk + ci_run zk clean --all + ci_run zk run yarn + ci_run cp etc/tokens/{test,localhost}.json + ci_run zk compiler all + ci_run zk contract build + ci_run zk f yarn run l2-contracts build + ci_run curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^26.key + + - name: update-images + env: + DOCKER_ACTION: ${{ inputs.action }} + COMPONENT: ${{ matrix.component }} + RUST_NIGHTLY_COMPONENTS: | + ( + "witness-generator" + "prover-v2" + "circuit-synthesizer" + "prover-fri" + "prover-gpu-fri" + "witness-vector-generator" + "prover-fri-gateway" + "proof-fri-compressor" + ) + run: | + nightly_components=${{env.RUST_NIGHTLY_COMPONENTS}} + ci_run docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }} + ci_run gcloud auth configure-docker us-docker.pkg.dev,asia-docker.pkg.dev -q + + if [[ " ${nightly_components[*]} " =~ " ${COMPONENT} " ]]; then + underscored_name=$(echo "$COMPONENT" | tr '-' '_') + # Hack: prover-v2 codebase located in prover directory instead of prover_v2 + if [ "$underscored_name" == "prover_v2" ]; then + underscored_name="prover" + fi + # prover-gpu-fri codebase located in prover_fri directory instead of prover_gpu_fri + if [ "$underscored_name" == "prover_gpu_fri" ]; then + underscored_name="prover_fri" + fi + ci_run rustup default nightly-2023-07-21 + ci_run echo [workspace] > Cargo.toml + ci_run echo members = [\"prover/${underscored_name}\"] >> Cargo.toml + ci_run cp prover/Cargo.lock Cargo.lock + ci_run zk docker $DOCKER_ACTION $COMPONENT + else + ci_run zk docker $DOCKER_ACTION $COMPONENT + fi + + - name: Show sccache stats + if: always() + run: | + ci_run sccache --show-stats + ci_run cat /tmp/sccache_log.txt diff --git a/.github/workflows/check-pr-title.yml b/.github/workflows/check-pr-title.yml new file mode 100644 index 00000000000..02c9b48600a --- /dev/null +++ b/.github/workflows/check-pr-title.yml @@ -0,0 +1,18 @@ +name: Check PR title +on: + pull_request_target: + types: + - opened + - reopened + - edited + - synchronize + +jobs: + lint: + runs-on: ubuntu-latest + permissions: + statuses: write + steps: + - uses: aslafy-z/conventional-pr-title-action@v3 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml new file mode 100644 index 00000000000..7665f06aaca --- /dev/null +++ b/.github/workflows/ci-core-reusable.yml @@ -0,0 +1,326 @@ +name: Workflow template for CI jobs for Core Components +on: + workflow_call: + +jobs: + lint: + runs-on: [matterlabs-ci-runner] + + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + with: + submodules: "recursive" + + - name: Setup environment + run: | + echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV + echo $(pwd)/bin >> $GITHUB_PATH + echo IN_DOCKER=1 >> .env + + - name: Start services + run: | + docker-compose -f docker-compose-runner.yml pull + docker-compose -f docker-compose-runner.yml up --build -d zk + ci_run sccache --start-server + + - name: Setup db + run: | + ci_run zk + ci_run zk db migrate + + - name: Lints + run: | + ci_run zk fmt --check + ci_run zk lint rust --check + ci_run zk lint js --check + ci_run zk lint ts --check + ci_run zk lint md --check + ci_run zk db check-sqlx-data + + unit-tests: + runs-on: [matterlabs-ci-runner] + + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + with: + submodules: "recursive" + + - name: Setup environment + run: | + echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV + echo $(pwd)/bin >> $GITHUB_PATH + echo IN_DOCKER=1 >> .env + + - name: Start services + run: | + docker-compose -f docker-compose-runner.yml pull + docker-compose -f docker-compose-runner.yml up --build -d geth zk postgres + ci_run sccache --start-server + + - name: Init + run: | + ci_run zk + ci_run zk run yarn + ci_run zk db setup + ci_run zk compiler all + ci_run zk contract build + ci_run zk run yarn + + - name: Contracts unit tests + run: ci_run zk test l1-contracts + + - name: Rust unit tests + run: ci_run zk test rust + + loadtest: + runs-on: [matterlabs-ci-runner] + + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + with: + submodules: "recursive" + + - name: Setup environment + run: | + echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV + echo $(pwd)/bin >> $GITHUB_PATH + echo IN_DOCKER=1 >> .env + + - name: Loadtest configuration + run: | + echo EXPECTED_TX_COUNT="16000" >> .env + echo FAIL_FAST=true >> .env + echo IN_DOCKER=1 >> .env + echo DATABASE_MERKLE_TREE_MODE=lightweight >> .env + + - name: Start services + run: | + docker-compose -f docker-compose-runner.yml pull + docker-compose -f docker-compose-runner.yml up --build -d geth zk postgres + ci_run sccache --start-server + + - name: Init + run: | + ci_run git config --global --add safe.directory /usr/src/zksync + ci_run git config --global --add safe.directory /usr/src/zksync/sdk/binaryen + ci_run git config --global --add safe.directory /usr/src/zksync/etc/system-contracts + ci_run git config --global --add safe.directory /usr/src/zksync/contracts + + ci_run zk + ci_run zk init + + # `sleep 30` because we need to wait until server added all the tokens + - name: Run server + run: | + ci_run zk server --uring --components api,tree,eth,data_fetcher,state_keeper,housekeeper &>server.log & + ci_run sleep 30 + + - name: Perform loadtest + run: ci_run zk run loadtest + + - name: Show logs + if: always() + run: | + ci_run cat server.log + ci_run sccache --show-stats + ci_run cat /tmp/sccache_log.txt + + integration: + runs-on: [matterlabs-ci-runner] + + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + with: + submodules: "recursive" + + - name: Setup environment + run: | + echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV + echo $(pwd)/bin >> $GITHUB_PATH + echo IN_DOCKER=1 >> .env + echo RUN_CONTRACT_VERIFICATION_TEST=true >> .env + echo ZKSYNC_DEBUG_LOGS=true >> .env + + - name: Download zksolc/solc and zkvyper/vyper + run: | + sudo apt update && sudo apt install wget -y + + mkdir -p $(pwd)/etc/solc-bin/0.8.21 + wget https://github.com/ethereum/solc-bin/raw/gh-pages/linux-amd64/solc-linux-amd64-v0.8.21%2Bcommit.d9974bed + mv solc-linux-amd64-v0.8.21+commit.d9974bed $(pwd)/etc/solc-bin/0.8.21/solc + chmod +x $(pwd)/etc/solc-bin/0.8.21/solc + + mkdir -p $(pwd)/etc/zksolc-bin/v1.3.14 + wget https://github.com/matter-labs/zksolc-bin/raw/main/linux-amd64/zksolc-linux-amd64-musl-v1.3.14 + mv zksolc-linux-amd64-musl-v1.3.14 $(pwd)/etc/zksolc-bin/v1.3.14/zksolc + chmod +x $(pwd)/etc/zksolc-bin/v1.3.14/zksolc + + mkdir -p $(pwd)/etc/vyper-bin/0.3.3 + wget -O vyper0.3.3 https://github.com/vyperlang/vyper/releases/download/v0.3.3/vyper.0.3.3%2Bcommit.48e326f0.linux + mv vyper0.3.3 $(pwd)/etc/vyper-bin/0.3.3/vyper + chmod +x $(pwd)/etc/vyper-bin/0.3.3/vyper + + mkdir -p $(pwd)/etc/zkvyper-bin/v1.3.10 + wget https://github.com/matter-labs/zkvyper-bin/raw/main/linux-amd64/zkvyper-linux-amd64-musl-v1.3.10 + mv zkvyper-linux-amd64-musl-v1.3.10 $(pwd)/etc/zkvyper-bin/v1.3.10/zkvyper + chmod +x $(pwd)/etc/zkvyper-bin/v1.3.10/zkvyper + + - name: Start services + run: | + docker-compose -f docker-compose-runner.yml pull + docker-compose -f docker-compose-runner.yml up --build -d geth zk postgres + ci_run sccache --start-server + + - name: Init + run: | + ci_run git config --global --add safe.directory /usr/src/zksync + ci_run git config --global --add safe.directory /usr/src/zksync/sdk/binaryen + ci_run git config --global --add safe.directory /usr/src/zksync/etc/system-contracts + ci_run git config --global --add safe.directory /usr/src/zksync/contracts + ci_run zk + ci_run zk init + + + # `sleep 5` because we need to wait until server started properly + - name: Run server + run: | + ci_run zk server &>server.log & + ci_run sleep 5 + + - name: Run contract verifier + run: | + ci_run zk contract_verifier &>contract_verifier.log & + ci_run sleep 2 + + - name: Server integration tests + run: ci_run zk test i server + + - name: Fee projection tests + run: ci_run zk test i fees + + - name: Run revert test + run: | + ci_run pkill zksync_server || true + ci_run sleep 2 + ci_run zk test i revert + + # This test should be the last one as soon as it + # finished bootloader will be different + - name: Run upgrade test + run: | + ci_run pkill zksync_server || true + ci_run sleep 10 + ci_run zk test i upgrade + + - name: Show logs + if: always() + run: | + ci_run cat server.log + ci_run cat contract_verifier.log + ci_run cat core/tests/revert-test/revert.log + ci_run cat core/tests/upgrade-test/upgrade.log + ci_run sccache --show-stats + ci_run cat /tmp/sccache_log.txt + + external-node: + runs-on: [matterlabs-ci-runner] + + steps: + - name: Checkout code # Checks out the repository under $GITHUB_WORKSPACE, so the job can access it. + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + with: + submodules: "recursive" + + - name: Setup environment + run: | + echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV + echo $(pwd)/bin >> $GITHUB_PATH + echo IN_DOCKER=1 >> .env + echo RUN_CONTRACT_VERIFICATION_TEST=true >> .env + echo ZKSYNC_DEBUG_LOGS=true >> .env + + - name: Download zksolc/solc and zkvyper/vyper + run: | + sudo apt update && sudo apt install wget -y + + mkdir -p $(pwd)/etc/solc-bin/0.8.21 + wget https://github.com/ethereum/solc-bin/raw/gh-pages/linux-amd64/solc-linux-amd64-v0.8.21%2Bcommit.d9974bed + mv solc-linux-amd64-v0.8.21+commit.d9974bed $(pwd)/etc/solc-bin/0.8.21/solc + chmod +x $(pwd)/etc/solc-bin/0.8.21/solc + + mkdir -p $(pwd)/etc/zksolc-bin/v1.3.14 + wget https://github.com/matter-labs/zksolc-bin/raw/main/linux-amd64/zksolc-linux-amd64-musl-v1.3.14 + mv zksolc-linux-amd64-musl-v1.3.14 $(pwd)/etc/zksolc-bin/v1.3.14/zksolc + chmod +x $(pwd)/etc/zksolc-bin/v1.3.14/zksolc + + mkdir -p $(pwd)/etc/vyper-bin/0.3.3 + wget -O vyper0.3.3 https://github.com/vyperlang/vyper/releases/download/v0.3.3/vyper.0.3.3%2Bcommit.48e326f0.linux + mv vyper0.3.3 $(pwd)/etc/vyper-bin/0.3.3/vyper + chmod +x $(pwd)/etc/vyper-bin/0.3.3/vyper + + mkdir -p $(pwd)/etc/zkvyper-bin/v1.3.10 + wget https://github.com/matter-labs/zkvyper-bin/raw/main/linux-amd64/zkvyper-linux-amd64-musl-v1.3.10 + mv zkvyper-linux-amd64-musl-v1.3.10 $(pwd)/etc/zkvyper-bin/v1.3.10/zkvyper + chmod +x $(pwd)/etc/zkvyper-bin/v1.3.10/zkvyper + + - name: Start services + run: | + docker-compose -f docker-compose-runner.yml pull + docker-compose -f docker-compose-runner.yml up --build -d geth zk postgres + ci_run sccache --start-server + + - name: Init + run: | + ci_run git config --global --add safe.directory /usr/src/zksync + ci_run git config --global --add safe.directory /usr/src/zksync/sdk/binaryen + ci_run git config --global --add safe.directory /usr/src/zksync/etc/system-contracts + ci_run git config --global --add safe.directory /usr/src/zksync/contracts + ci_run zk + ci_run zk init + + # `sleep 30` because we need to wait until server started properly + - name: Run server + run: | + ci_run zk server &>server.log & + ci_run sleep 30 + + - name: Run external node + run: | + ci_run zk env ext-node-docker + ci_run zk db setup + ci_run zk external-node &>ext-node.log & + ci_run sleep 30 + + - name: Integration tests + run: ci_run zk test i server --testPathIgnorePatterns 'contract-verification' + + - name: Run Cross EN Checker + run: ci_run zk run cross-en-checker + + # - name: Run revert test + # run: | + # ci_run pkill zksync_server || true + # ci_run sleep 2 + # ci_run zk test i revert + # # Check that the rollback was performed on the EN + # ci_run grep -q 'Rollback successfully completed' ext-node.log + # # Restart the EN + # ci_run zk server &>>server.log & + # ci_run sleep 30 + # ZKSYNC_ENV=ext-node-docker ci_run zk external-node &>>ext-node.log & + # ci_run sleep 30 + + - name: Run upgrade test + run: | + ci_run zk env docker + CHECK_EN_URL="http://0.0.0.0:3060" ci_run zk test i upgrade + + - name: Show logs + if: always() + run: | + ci_run cat server.log + ci_run cat ext-node.log + # ci_run cat core/tests/revert-test/revert.log + ci_run cat core/tests/upgrade-test/upgrade.log + ci_run sccache --show-stats + ci_run cat /tmp/sccache_log.txt diff --git a/.github/workflows/ci-prover-reusable.yml b/.github/workflows/ci-prover-reusable.yml new file mode 100644 index 00000000000..426855cdaa2 --- /dev/null +++ b/.github/workflows/ci-prover-reusable.yml @@ -0,0 +1,55 @@ +name: Workflow template for CI jobs for Prover Components +on: + workflow_call: + +jobs: + unit-tests: + runs-on: [matterlabs-ci-runner] + + steps: + - name: Prepare environment + run: | + sudo apt update && sudo apt install -y \ + pkg-config libclang-dev build-essential lldb lld \ + clang openssl libssl-dev gcc g++ pkg-config libclang-dev curl wget + + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + with: + submodules: "recursive" + + - name: Use Node.js 18.18.0 + uses: actions/setup-node@v2 + with: + node-version: '18.18.0' + + - name: Install Rust nightly-2023-07-21 + uses: actions-rs/toolchain@v1 + with: + toolchain: nightly-2023-07-21 + override: true + + - name: Setup environment + run: | + echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV + echo $(pwd)/bin >> $GITHUB_PATH + echo IN_DOCKER=1 >> .env + npm install -g yarn + cargo install sqlx-cli --version 0.5.13 + + - name: Start services + run: | + docker-compose -f docker-compose-runner.yml pull + docker-compose -f docker-compose-runner.yml up --build -d zk + ci_run sccache --start-server + + - name: Init + run: | + zk + zk config compile + zk db setup + + - name: Prover unit tests + run: | + cd prover + # Not all tests are enabled since, prover and setup_key_generator_and_server requires bellman-cuda to be present + zk f cargo +nightly-2023-07-21 test --release -p zksync_witness_generator -p vk_setup_data_generator_server_fri -p zksync_prover_fri -p zksync_witness_vector_generator -p zksync_prover_fri_utils diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000..633680bf2d6 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,113 @@ +name: CI + +on: + pull_request: + merge_group: + push: + branches: + - staging + - trying + - '!release-please--branches--**' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + changed_files: + runs-on: [matterlabs-default-infra-runners] + name: Test changed-files + outputs: + core: ${{ steps.changed-files-yaml.outputs.core_any_changed }} + prover: ${{ steps.changed-files-yaml.outputs.prover_any_changed }} + all: ${{ steps.changed-files-yaml.outputs.all_any_changed }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + submodules: "recursive" + + - name: Get all test, doc and src files that have changed + id: changed-files-yaml + uses: tj-actions/changed-files@v37 + with: + files_yaml: | + # If you want to exclude some files, please adjust here. + prover: + - prover/** + - '!.github/release-please/manifest.json' + core: + - core/** + all: + - '!core/**' + - '!prover/**' + ci-for-core: + name: CI for Core Components + needs: changed_files + if: needs.changed_files.outputs.core == 'true' || needs.changed_files.outputs.all == 'true' + uses: ./.github/workflows/ci-core-reusable.yml + + ci-for-prover: + needs: changed_files + if: needs.changed_files.outputs.prover == 'true' || needs.changed_files.outputs.all == 'true' + name: CI for Prover Components + uses: ./.github/workflows/ci-prover-reusable.yml + + build-core-images: + name: Build core images + needs: changed_files + if: needs.changed_files.outputs.core == 'true' || needs.changed_files.outputs.all == 'true' + uses: ./.github/workflows/build-core-template.yml + with: + image_tag: ${{ needs.setup.outputs.image_tag }} + image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }} + action: "build" + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + + build-prover-images: + name: Build prover images + needs: changed_files + if: needs.changed_files.outputs.prover == 'true' || needs.changed_files.outputs.all == 'true' + uses: ./.github/workflows/build-prover-template.yml + with: + image_tag: ${{ needs.setup.outputs.image_tag }} + image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }} + action: "build" + ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }} + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + + ci-success: + name: Github Status Check + runs-on: ubuntu-latest + if: always() && !cancelled() + needs: [ci-for-core, ci-for-prover, build-core-images, build-prover-images] + steps: + - name: Status + run: | + # This will check all jobs status in the `needs` list, and fail job if one is failed. + # Since we split prover and core to different flows, this job will be only as Required Status Check in the Pull Request. + if [[ ${{ contains(join(needs.*.result, ','), 'failure') }} == "true" ]]; then + echo "Intentionally failing to block PR from merging" + exit 1 + fi + + notify: + if: always() && !cancelled() + name: Notify on failures + runs-on: ubuntu-latest + needs: [ci-for-core, ci-for-prover, build-core-images, build-prover-images] + steps: + - uses: technote-space/workflow-conclusion-action@45ce8e0eb155657ab8ccf346ade734257fd196a5 # v3 + - name: Notify to Mattermost (on incidents) + uses: tferreira/matterfy@releases/v1 + if: ${{ env.WORKFLOW_CONCLUSION == 'failure' }} + with: + type: failure + job_name: "*CI suites for PR #${{ github.event.pull_request.number }} failed*" + icon_emoji: octocat + channel: "ci-notifications" + url: ${{ secrets.MATTERMOST_WEBHOOK }} diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 00000000000..5064bfe2272 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,47 @@ +name: coverage + +on: + push: + branches: + - "**" + +jobs: + generate: + runs-on: [self-hosted, ci-runner] + + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + with: + submodules: "recursive" + + - name: Setup environment + run: | + echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV + echo $(pwd)/bin >> $GITHUB_PATH + echo IN_DOCKER=1 >> .env + + - name: Start services + run: | + docker-compose -f docker-compose-runner.yml pull + docker-compose -f docker-compose-runner.yml up --build -d zk + ci_run sccache --start-server + + - name: Init + run: | + ci_run zk + ci_run zk run yarn + ci_run zk db drop + ci_run zk db setup + ci_run zk db reset-test + ci_run zk compiler all + ci_run zk contract build + + - name: Generate code coverage + run: ci_run zk f cargo tarpaulin + + - name: Upload to codecov.io + uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: "!target/sqlx" + fail_ci_if_error: true diff --git a/.github/workflows/vm-perf-comparison.yml b/.github/workflows/vm-perf-comparison.yml new file mode 100644 index 00000000000..3078c9bfa8a --- /dev/null +++ b/.github/workflows/vm-perf-comparison.yml @@ -0,0 +1,62 @@ +name: Compare VM perfomance to base branch + +on: + pull_request: + +jobs: + vm-benchmarks: + name: Run VM benchmarks + runs-on: [self-hosted, ci-runner] + + steps: + - uses: actions/checkout@v3 + with: + submodules: "recursive" + fetch-depth: 0 + ref: ${{ github.base_ref }} + + - name: checkout divergence point + run: git checkout $(git merge-base ${{ github.event.pull_request.head.sha }} HEAD) + + - name: setup-env + run: | + touch .env + echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV + echo $(pwd)/bin >> $GITHUB_PATH + + - name: init + run: | + docker-compose -f docker-compose-runner.yml up -d zk + + - name: run benchmarks on base branch + shell: bash + run: | + ci_run zk + ci_run zk compiler system-contracts + ci_run cargo bench --package vm-benchmark --bench iai | tee base-iai + + - name: checkout pull request + run: git checkout --force ${{ github.event.pull_request.head.sha }} + + - name: run benchmarks on PR + shell: bash + run: | + ci_run zk + ci_run zk compiler system-contracts + ci_run cargo bench --package vm-benchmark --bench iai | tee pr-iai + + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + echo "speedup<<$EOF" >> $GITHUB_OUTPUT + ci_run cargo run --package vm-benchmark --release --bin compare_iai_results base-iai pr-iai >> $GITHUB_OUTPUT + echo "$EOF" >> $GITHUB_OUTPUT + id: comparison + + - name: Comment on PR + uses: thollander/actions-comment-pull-request@v2 + with: + message: | + ${{ steps.comparison.outputs.speedup == '' && '## No performance difference detected (anymore)' || '## Detected VM performance changes' }} + ${{ steps.comparison.outputs.speedup }} + comment_tag: vm-performance-changes + mode: recreate + create_if_not_exists: ${{ steps.comparison.outputs.speedup != '' }} diff --git a/.github/workflows/zk-environment-cuda-12-0.publish.yml b/.github/workflows/zk-environment-cuda-12-0.publish.yml new file mode 100644 index 00000000000..f6471582bdf --- /dev/null +++ b/.github/workflows/zk-environment-cuda-12-0.publish.yml @@ -0,0 +1,50 @@ +name: publish zk-environment with cuda 12.0 docker image + +on: + pull_request: + branches: + - main + types: [closed] + paths: + - "docker/zk-environment-cuda-12-0/Dockerfile" + - ".github/workflows/zk-environment-cuda-12-0.publish.yml" + workflow_dispatch: + branches: + - "main" + +jobs: + push_to_registry: + if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' + name: Push Docker image to Docker Hub + runs-on: [self-hosted, ci-runner] + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + with: + submodules: "recursive" + + - name: Login to us-central1 GAR + run: | + gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev + + - name: Log in to Docker Hub + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and push + uses: docker/build-push-action@v4 + with: + context: . + file: docker/zk-environment-cuda-12-0/Dockerfile + push: true + target: nvidia-tools + tags: | + us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/zk-environment-cuda-12-0:latest + matterlabs/zk-environment:cuda-12-0-latest diff --git a/.github/workflows/zk-environment.publish.yml b/.github/workflows/zk-environment.publish.yml new file mode 100644 index 00000000000..561da619021 --- /dev/null +++ b/.github/workflows/zk-environment.publish.yml @@ -0,0 +1,53 @@ +name: publish zk-environment docker image + +on: + pull_request: + branches: + - main + types: [closed] + paths: + - "docker/zk-environment/Dockerfile" + - ".github/workflows/zk-environment.publish.yml" + workflow_dispatch: + branches: + - "main" + +jobs: + push_to_registry: + if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' + name: Push Docker image to Docker Hub + runs-on: [self-hosted, ci-runner] + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + with: + submodules: "recursive" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v2 + + - name: Log in to Docker Hub + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0 + with: + context: . + target: base + push: true + tags: "matterlabs/zk-environment:latest2.0-lightweight" + file: docker/zk-environment/Dockerfile + no-cache: true + + + - name: Build and push Docker image + uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0 + with: + context: . + target: nvidia-tools + push: true + tags: "matterlabs/zk-environment:latest2.0" + file: docker/zk-environment/Dockerfile + no-cache: true