From 30701fe50ad07447ab581f37dfccddf42d51b9e1 Mon Sep 17 00:00:00 2001 From: perekopskiy <53865202+perekopskiy@users.noreply.github.com> Date: Wed, 24 Apr 2024 11:13:11 +0300 Subject: [PATCH 1/2] ci: Separate contract verifier release (#1779) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Separates contract verifier release ## Checklist - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. - [ ] Spellcheck has been run via `zk spellcheck`. - [ ] Linkcheck has been run via `zk linkcheck`. --- .../build-contract-verifier-template.yml | 181 ++++++++++++++++++ .github/workflows/build-core-template.yml | 3 - .github/workflows/build-docker-from-tag.yml | 12 ++ .github/workflows/ci.yml | 13 ++ .github/workflows/release-test-stage.yml | 11 ++ 5 files changed, 217 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/build-contract-verifier-template.yml diff --git a/.github/workflows/build-contract-verifier-template.yml b/.github/workflows/build-contract-verifier-template.yml new file mode 100644 index 000000000000..2346efc96152 --- /dev/null +++ b/.github/workflows/build-contract-verifier-template.yml @@ -0,0 +1,181 @@ +name: Build contract verifier +on: + workflow_call: + secrets: + DOCKERHUB_USER: + description: "DOCKERHUB_USER" + required: true + DOCKERHUB_TOKEN: + description: "DOCKERHUB_TOKEN" + required: true + inputs: + image_tag_suffix: + description: "Optional suffix to override tag name generation" + type: string + required: false + action: + description: "Action with docker image" + type: string + default: "push" + required: false + compilers: + description: 'JSON of required compilers and their versions' + type: string + required: false + default: '[{ "zksolc": ["1.3.14", "1.3.16", "1.3.17", "1.3.1", "1.3.7", "1.3.18", "1.3.19", "1.3.21"] } , { "zkvyper": ["1.3.13"] }]' +jobs: + build-images: + name: Build and Push Docker Images + env: + IMAGE_TAG_SUFFIX: ${{ inputs.image_tag_suffix }} + runs-on: ${{ fromJSON('["matterlabs-ci-runner", "matterlabs-ci-runner-arm"]')[contains(matrix.platforms, 'arm')] }} + strategy: + matrix: + components: + - contract-verifier + platforms: + - linux/amd64 + + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + with: + submodules: "recursive" + + - name: setup-env + run: | + echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV + echo CI=1 >> $GITHUB_ENV + echo $(pwd)/bin >> $GITHUB_PATH + echo CI=1 >> .env + echo IN_DOCKER=1 >> .env + + - name: Download contracts + run: | + commit_sha=$(git submodule status contracts | awk '{print $1}' | tr -d '-') + page=1 + filtered_tag="" + while [ true ]; do + echo "Page: $page" + tags=$(curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/matter-labs/era-contracts/tags?per_page=100&page=${page}" | jq .) + if [ $(jq length <<<"$tags") -eq 0 ]; then + echo "No tag found on all pages." + echo "BUILD_CONTRACTS=true" >> "$GITHUB_ENV" + exit 0 + fi + filtered_tag=$(jq -r --arg commit_sha "$commit_sha" 'map(select(.commit.sha == $commit_sha)) | .[].name' <<<"$tags") + if [[ ! -z "$filtered_tag" ]]; then + echo "BUILD_CONTRACTS=false" >> "$GITHUB_ENV" + break + fi + ((page++)) + done + echo "Contracts tag is: ${filtered_tag}" + mkdir -p ./contracts + curl -s -LO https://github.com/matter-labs/era-contracts/releases/download/${filtered_tag}/l1-contracts.tar.gz + curl -s -LO https://github.com/matter-labs/era-contracts/releases/download/${filtered_tag}/l2-contracts.tar.gz + curl -s -LO https://github.com/matter-labs/era-contracts/releases/download/${filtered_tag}/system-contracts.tar.gz + tar -C ./contracts -zxf l1-contracts.tar.gz + tar -C ./contracts -zxf l2-contracts.tar.gz + tar -C ./contracts -zxf system-contracts.tar.gz + + - name: pre-download compilers + if: env.BUILD_CONTRACTS == 'true' + run: | + # Download needed versions of vyper compiler + # Not sanitized due to unconventional path and tags + mkdir -p ./hardhat-nodejs/compilers-v2/vyper/linux + wget -nv -O ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.10 https://github.com/vyperlang/vyper/releases/download/v0.3.10/vyper.0.3.10+commit.91361694.linux + wget -nv -O ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.3 https://github.com/vyperlang/vyper/releases/download/v0.3.3/vyper.0.3.3+commit.48e326f0.linux + chmod +x ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.10 + chmod +x ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.3 + + COMPILERS_JSON='${{ inputs.compilers }}' + echo "$COMPILERS_JSON" | jq -r '.[] | to_entries[] | .key as $compiler | .value[] | "\(.),\($compiler)"' | while IFS=, read -r version compiler; do + mkdir -p "./hardhat-nodejs/compilers-v2/$compiler" + wget -nv -O "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}" "https://github.com/matter-labs/${compiler}-bin/releases/download/v${version}/${compiler}-linux-amd64-musl-v${version}" + chmod +x "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}" + done + + - name: start-services + run: | + echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env + mkdir -p ./volumes/postgres + docker compose up -d zk postgres + ci_run sccache --start-server + + - name: init + run: | + ci_run git config --global --add safe.directory /usr/src/zksync + ci_run git config --global --add safe.directory /usr/src/zksync/sdk/binaryen + ci_run git config --global --add safe.directory /usr/src/zksync/contracts/system-contracts + ci_run git config --global --add safe.directory /usr/src/zksync/contracts + ci_run zk || true + ci_run yarn zk build + ci_run curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^26.key + + - name: build contracts + if: env.BUILD_CONTRACTS == 'true' + run: | + ci_run zk run yarn + ci_run cp etc/tokens/{test,localhost}.json + ci_run zk compiler all + ci_run zk contract build + ci_run zk f yarn run l2-contracts build + + - name: Login to Docker registries + if: ${{ inputs.action == 'push' }} + run: | + ci_run docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }} + ci_run gcloud auth configure-docker us-docker.pkg.dev -q + + - name: update-images + env: + DOCKER_ACTION: ${{ inputs.action }} + COMPONENT: ${{ matrix.components }} + PLATFORM: ${{ matrix.platforms }} + run: | + ci_run rustup default nightly-2023-08-21 + platform=$(echo $PLATFORM | tr '/' '-') + ci_run zk docker $DOCKER_ACTION --custom-tag=${IMAGE_TAG_SUFFIX} --platform=${PLATFORM} $COMPONENT + - name: Show sccache stats + if: always() + run: | + ci_run sccache --show-stats + ci_run cat /tmp/sccache_log.txt + + create_manifest: + name: Create release manifest + runs-on: matterlabs-ci-runner + needs: build-images + if: ${{ inputs.action == 'push' }} + strategy: + matrix: + component: + - name: contract-verifier + platform: linux/amd64 + env: + IMAGE_TAG_SUFFIX: ${{ inputs.image_tag_suffix }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + + - name: login to Docker registries + run: | + docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }} + gcloud auth configure-docker us-docker.pkg.dev -q + + - name: Create Docker manifest + run: | + docker_repositories=("matterlabs/${{ matrix.component.name }}" "us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component.name }}") + platforms=${{ matrix.component.platform }} + for repo in "${docker_repositories[@]}"; do + platform_tags="" + for platform in ${platforms//,/ }; do + platform=$(echo $platform | tr '/' '-') + platform_tags+=" --amend ${repo}:${IMAGE_TAG_SUFFIX}-${platform}" + done + for manifest in "${repo}:${IMAGE_TAG_SUFFIX}" "${repo}:2.0-${IMAGE_TAG_SUFFIX}" "${repo}:latest" "${repo}:latest2.0"; do + docker manifest create ${manifest} ${platform_tags} + docker manifest push ${manifest} + done + done diff --git a/.github/workflows/build-core-template.yml b/.github/workflows/build-core-template.yml index c0da3f71d1b7..cd205160bd80 100644 --- a/.github/workflows/build-core-template.yml +++ b/.github/workflows/build-core-template.yml @@ -34,7 +34,6 @@ jobs: components: - server-v2 - external-node - - contract-verifier - snapshots-creator platforms: - linux/amd64 @@ -162,8 +161,6 @@ jobs: platform: linux/amd64 - name: external-node platform: linux/amd64,linux/arm64 - - name: contract-verifier - platform: linux/amd64 - name: snapshots-creator platform: linux/amd64 env: diff --git a/.github/workflows/build-docker-from-tag.yml b/.github/workflows/build-docker-from-tag.yml index 925d03c1a9dc..103490ef4b23 100644 --- a/.github/workflows/build-docker-from-tag.yml +++ b/.github/workflows/build-docker-from-tag.yml @@ -16,6 +16,7 @@ on: tags: - core-v** - prover-v** + - contract_verifier-v** concurrency: docker-build @@ -57,6 +58,17 @@ jobs: with: image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }} + build-push-contract-verifier: + name: Build and push image + needs: [ setup ] + uses: ./.github/workflows/build-contract-verifier-template.yml + if: contains(github.ref_name, 'contract_verifier') + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + with: + image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }} + build-push-prover-images: name: Build and push image needs: [setup] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 471280a7b9f7..1d5f4a474c7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,7 @@ jobs: - 'docker/external-node/**' - 'docker/server/**' - '.github/workflows/build-core-template.yml' + - '.github/workflows/build-contract-verifier-template.yml' - '.github/workflows/ci-core-reusable.yml' - '.github/workflows/ci-core-lint-reusable.yml' - 'Cargo.toml' @@ -115,6 +116,18 @@ jobs: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + build-contract-verifier: + name: Build contract verifier + needs: changed_files + if: ${{ (needs.changed_files.outputs.core == 'true' || needs.changed_files.outputs.all == 'true') && !contains(github.ref_name, 'release-please--branches') }} + uses: ./.github/workflows/build-contract-verifier-template.yml + with: + image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }} + action: "build" + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + build-prover-images: name: Build prover images needs: changed_files diff --git a/.github/workflows/release-test-stage.yml b/.github/workflows/release-test-stage.yml index 7627752d5af2..c54d84565015 100644 --- a/.github/workflows/release-test-stage.yml +++ b/.github/workflows/release-test-stage.yml @@ -71,6 +71,17 @@ jobs: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + build-push-contract-verifier: + name: Build and push images + needs: [ setup, changed_files ] + uses: ./.github/workflows/build-contract-verifier-template.yml + if: needs.changed_files.outputs.core == 'true' || needs.changed_files.outputs.all == 'true' + with: + image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }} + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + build-push-prover-images: name: Build and push images needs: [setup, changed_files] From 884882039c54fd301be09796c8bdd783f2c866e2 Mon Sep 17 00:00:00 2001 From: kelemeno <34402761+kelemeno@users.noreply.github.com> Date: Wed, 24 Apr 2024 09:32:06 +0100 Subject: [PATCH 2/2] fix: migration fix 3 ( enought to merge to main later) (#1771) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ ## Why ❔ ## Checklist - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. - [ ] Spellcheck has been run via `zk spellcheck`. - [ ] Linkcheck has been run via `zk linkcheck`. --------- Signed-off-by: Danil Co-authored-by: Bence Haromi Co-authored-by: Jmunoz Co-authored-by: Lyova Potyomkin Co-authored-by: Stanislav Breadless Co-authored-by: Bence Haromi <56651250+benceharomi@users.noreply.github.com> Co-authored-by: Javier Chatruc Co-authored-by: Francisco Krause Arnim Co-authored-by: Javier Rodríguez Chatruc <49622509+jrchatruc@users.noreply.github.com> Co-authored-by: Francisco Krause Arnim <56402156+fkrause98@users.noreply.github.com> Co-authored-by: Santiago Pittella Co-authored-by: Danil Co-authored-by: Leandro Ferrigno Co-authored-by: Raid Ateir Co-authored-by: fborello-lambda Co-authored-by: mm Co-authored-by: Marcin M <128217157+mm-zk@users.noreply.github.com> Co-authored-by: perekopskiy <53865202+perekopskiy@users.noreply.github.com> Co-authored-by: Alex Ostrovski Co-authored-by: Raid5594 <52794079+Raid5594@users.noreply.github.com> --- contracts | 2 +- .../protocol-upgrade/src/hyperchain-upgrade.ts | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/contracts b/contracts index 6e9af1a1835d..501691051a74 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit 6e9af1a1835d871c57bff041bd0cd1a6992c1ec5 +Subproject commit 501691051a745bdb03922e57f263497a866701dd diff --git a/infrastructure/protocol-upgrade/src/hyperchain-upgrade.ts b/infrastructure/protocol-upgrade/src/hyperchain-upgrade.ts index 4d5cb98450a3..ce7d434d3af3 100644 --- a/infrastructure/protocol-upgrade/src/hyperchain-upgrade.ts +++ b/infrastructure/protocol-upgrade/src/hyperchain-upgrade.ts @@ -134,11 +134,22 @@ async function hyperchainFullUpgrade() { 'cp etc/env/.init.env etc/env/l1-inits/.init.env && rm ./etc/env/l2-inits/zksync_local.init.env && rm ./etc/env/target/zksync_local.env' ); await spawn('zk env zksync_local'); + env.reload('zksync_local'); env.modify( 'CONTRACTS_ERA_DIAMOND_PROXY_ADDR', process.env.CONTRACTS_DIAMOND_PROXY_ADDR, `etc/env/l1-inits/${process.env.L1_ENV_NAME ? process.env.L1_ENV_NAME : '.init'}.env` ); + env.modify( + 'CONTRACTS_L2_SHARED_BRIDGE_ADDR', + process.env.CONTRACTS_L2_ERC20_BRIDGE_ADDR, + `etc/env/l2-inits/${process.env.ZKSYNC_ENV}.init.env` + ); + env.modify( + 'CONTRACTS_BASE_TOKEN_ADDR', + '0x0000000000000000000000000000000000000001', + `etc/env/l2-inits/${process.env.ZKSYNC_ENV}.init.env` + ); await deploySharedBridgeL2Implementation(); @@ -170,7 +181,7 @@ async function hyperchainFullUpgrade() { async function postPropose() { // we need to set up the prover dal await setupForDal(DalPath.ProverDal, process.env.DATABASE_PROVER_URL); - await 'zk db migrate'; + await spawn('zk db migrate'); } export const command = new Command('hyperchain-upgrade').description('create and publish custom l2 upgrade');