Merge branch 'main' into evm-equivalence-yul-new

lambdaclass · Sep 4, 2024 · 9eea1c0 · 9eea1c0
2 parents bf54e4d + 9821a20
commit 9eea1c0
Show file tree

Hide file tree

Showing 170 changed files with 1,591 additions and 2,251 deletions.
diff --git a/.github/workflows/build-prover-fri-gpu-gar.yml b/.github/workflows/build-prover-fri-gpu-gar.yml
@@ -34,13 +34,13 @@ jobs:
           gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
 
       - name: Build and push
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
         with:
           context: docker/prover-gpu-fri-gar
           build-args: |

diff --git a/.github/workflows/build-prover-template.yml b/.github/workflows/build-prover-template.yml
@@ -180,7 +180,7 @@ jobs:
           - witness-vector-generator
     steps:
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
 
       - name: Login to us-central1 GAR
         run: |

diff --git a/.github/workflows/build-tee-prover-template.yml b/.github/workflows/build-tee-prover-template.yml
@@ -28,15 +28,15 @@ jobs:
       IMAGE_TAG_SUFFIX: ${{ inputs.image_tag_suffix }}
     runs-on: [matterlabs-ci-runner-high-performance]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
         if: ${{ github.event_name == 'workflow_dispatch' }}
         with:
           ref: ${{ github.event.inputs.target_branch }}
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
         if: ${{ github.event_name != 'workflow_dispatch' }}
 
-      - uses: cachix/install-nix-action@v27
+      - uses: cachix/install-nix-action@ba0dd844c9180cbf77aa72a116d6fbc515d0e87b # v27
         with:
           extra_nix_config: |
             access-tokens = github.com=${{ github.token }}
@@ -45,7 +45,7 @@ jobs:
             sandbox = true
 
       - name: Setup Attic cache
-        uses: ryanccn/attic-action@v0
+        uses: ryanccn/attic-action@618a980988d704a7709daeea88526146acd1d45f # v0.2.1
         with:
           endpoint: https://attic.teepot.org/
           cache: tee-pot

diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml
@@ -52,22 +52,19 @@ jobs:
 
       - name: Init
         run: |
-          ci_run zk
           ci_run run_retried rustup show
-          ci_run zk run yarn
-          ci_run zk db setup
-          ci_run zk compiler all
-          ci_run zk contract build
+          ci_run ./bin/zkt
+          ci_run zk_supervisor contracts
 
       - name: Contracts unit tests
         run: ci_run yarn l1-contracts test
 
       - name: Rust unit tests
         run: |
-          ci_run zk test rust
+          ci_run zk_supervisor test rust
           # Benchmarks are not tested by `cargo nextest` unless specified explicitly, and even then `criterion` harness is incompatible
           # with how `cargo nextest` runs tests. Thus, we run criterion-based benchmark tests manually.
-          ci_run zk f cargo test --release -p vm-benchmark --bench oneshot --bench batch
+          ci_run cargo test --release -p vm-benchmark --bench oneshot --bench batch
 
   loadtest:
     runs-on: [ matterlabs-ci-runner-high-performance ]
@@ -90,8 +87,10 @@ jobs:
 
       - name: Loadtest configuration
         run: |
-          echo EXPECTED_TX_COUNT=${{ matrix.vm_mode == 'new' && 22000 || 16000 }} >> .env
-          echo ACCOUNTS_AMOUNT="150" >> .env
+          echo EXPECTED_TX_COUNT=${{ matrix.vm_mode == 'new' && 21000 || 16000 }} >> .env
+          echo ACCOUNTS_AMOUNT="100" >> .env
+          echo MAX_INFLIGHT_TXS="10" >> .env
+          echo SYNC_API_REQUESTS_LIMIT="15" >> .env
           echo FAIL_FAST=true >> .env
           echo IN_DOCKER=1 >> .env
           echo DATABASE_MERKLE_TREE_MODE=lightweight >> .env
@@ -115,7 +114,8 @@ jobs:
       - name: Run server
         run: |
           EXPERIMENTAL_VM_STATE_KEEPER_FAST_VM_MODE=${{ matrix.vm_mode }} \
-          PASSED_ENV_VARS="EXPERIMENTAL_VM_STATE_KEEPER_FAST_VM_MODE" \
+          CHAIN_MEMPOOL_DELAY_INTERVAL=50 \
+          PASSED_ENV_VARS="EXPERIMENTAL_VM_STATE_KEEPER_FAST_VM_MODE,CHAIN_MEMPOOL_DELAY_INTERVAL" \
             ci_run zk server --uring --components api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads &>server.log &
           ci_run sleep 60
 

diff --git a/.github/workflows/ci-zk-toolbox-reusable.yml b/.github/workflows/ci-zk-toolbox-reusable.yml
@@ -249,7 +249,7 @@ jobs:
 
 
       - name: Upload logs
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
         if: always()
         with:
           name: logs

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -31,7 +31,7 @@ jobs:
 
       - name: Get changed files
         id: changed-files
-        uses: tj-actions/changed-files@v39
+        uses: tj-actions/changed-files@fea790cb660e33aef4bdf07304e28fedd77dfa13 # v39
         with:
           files_yaml: |
             prover:

diff --git a/.github/workflows/nodejs-license.yaml b/.github/workflows/nodejs-license.yaml
@@ -47,7 +47,7 @@ jobs:
         uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
 
       - name: Use Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3
         with:
           node-version: 18
 

diff --git a/.github/workflows/protobuf.yaml b/.github/workflows/protobuf.yaml
@@ -33,7 +33,7 @@ jobs:
   compatibility:
     runs-on: [ubuntu-22.04-github-hosted-16core]
     steps:
-      - uses: mozilla-actions/[email protected].3
+      - uses: mozilla-actions/sccache-action@89e9040de88b577a072e3760aaf59f585da083af # v0.0.5
 
       # before
       - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
@@ -69,7 +69,7 @@ jobs:
           | xargs cat > ./after.binpb
 
       # compare
-      - uses: bufbuild/buf-setup-action@v1
+      - uses: bufbuild/buf-setup-action@54abbed4fe8d8d45173eca4798b0c39a53a7b658 # v1.39.0
         with:
           github_token: ${{ github.token }}
       - name: buf breaking

diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml
@@ -15,7 +15,7 @@ jobs:
     steps:
       - name: Run release-please
         id: release
-        uses: google-github-actions/release-please-action@v4
+        uses: google-github-actions/release-please-action@e4dc86ba9405554aeba3c6bb2d169500e7d3b4ee # v4.1.1
         with:
             token: ${{ secrets.RELEASE_TOKEN }}
             config-file: .github/release-please/config.json

diff --git a/.github/workflows/release-test-stage.yml b/.github/workflows/release-test-stage.yml
@@ -23,7 +23,7 @@ jobs:
 
       - name: Get all test, doc and src files that have changed
         id: changed-files-yaml
-        uses: tj-actions/changed-files@v37
+        uses: tj-actions/changed-files@fea790cb660e33aef4bdf07304e28fedd77dfa13 # v39
         with:
           files_yaml: |
             # TODO: make it more granular, as already implemented in CI workflow

diff --git a/.github/workflows/vm-perf-comparison.yml b/.github/workflows/vm-perf-comparison.yml
@@ -68,7 +68,7 @@ jobs:
         id: comparison
 
       - name: Comment on PR
-        uses: thollander/actions-comment-pull-request@v2
+        uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6 # v2.5.0
         with:
           message: |
             ${{ steps.comparison.outputs.speedup == '' && '## No performance difference detected (anymore)' || '## Detected VM performance changes' }}

diff --git a/.github/workflows/zk-environment-publish.yml b/.github/workflows/zk-environment-publish.yml
@@ -36,7 +36,7 @@ jobs:
 
       - name: Get changed files
         id: changed-files-yaml
-        uses: tj-actions/changed-files@v39
+        uses: tj-actions/changed-files@fea790cb660e33aef4bdf07304e28fedd77dfa13 # v39
         with:
           files_yaml: |
             zk_env:

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -79,8 +79,6 @@ members = [
     "core/tests/test_account",
     "core/tests/loadnext",
     "core/tests/vm-benchmark",
-    # Parts of prover workspace that are needed for Core workspace
-    "prover/crates/lib/prover_dal",
 ]
 resolver = "2"
 
@@ -217,7 +215,7 @@ zk_evm_1_4_1 = { package = "zk_evm", version = "0.141.0" }
 zk_evm_1_5_0 = { package = "zk_evm", version = "=0.150.4" }
 
 # New VM; pinned to a specific commit because of instability
-vm2 = { git = "https://github.com/matter-labs/vm2.git", rev = "2276b7b5af520fca0477bdafe43781b51896d235" }
+vm2 = { git = "https://github.com/matter-labs/vm2.git", rev = "4ef15d46410ffc11744771a3a6c7c09dd9470c90" }
 
 # Consensus dependencies.
 zksync_concurrency = "=0.1.0-rc.11"
@@ -233,7 +231,6 @@ zksync_protobuf_build = "=0.1.0-rc.11"
 
 # "Local" dependencies
 zksync_multivm = { version = "0.1.0", path = "core/lib/multivm" }
-zksync_prover_dal = { version = "0.1.0", path = "prover/crates/lib/prover_dal" }
 zksync_vlog = { version = "0.1.0", path = "core/lib/vlog" }
 zksync_vm_interface = { version = "0.1.0", path = "core/lib/vm_interface" }
 zksync_vm_executor = { version = "0.1.0", path = "core/lib/vm_executor" }

diff --git a/core/bin/zksync_server/src/node_builder.rs b/core/bin/zksync_server/src/node_builder.rs
@@ -122,7 +122,6 @@ impl MainNodeBuilder {
         let pools_layer = PoolsLayerBuilder::empty(config, secrets)
             .with_master(true)
             .with_replica(true)
-            .with_prover(true) // Used by house keeper.
             .build();
         self.node.add_layer(pools_layer);
         Ok(self)
@@ -446,18 +445,9 @@ impl MainNodeBuilder {
 
     fn add_house_keeper_layer(mut self) -> anyhow::Result<Self> {
         let house_keeper_config = try_load_config!(self.configs.house_keeper_config);
-        let fri_prover_config = try_load_config!(self.configs.prover_config);
-        let fri_witness_generator_config = try_load_config!(self.configs.witness_generator_config);
-        let fri_prover_group_config = try_load_config!(self.configs.prover_group_config);
-        let fri_proof_compressor_config = try_load_config!(self.configs.proof_compressor_config);
-
-        self.node.add_layer(HouseKeeperLayer::new(
-            house_keeper_config,
-            fri_prover_config,
-            fri_witness_generator_config,
-            fri_prover_group_config,
-            fri_proof_compressor_config,
-        ));
+
+        self.node
+            .add_layer(HouseKeeperLayer::new(house_keeper_config));
 
         Ok(self)
     }

diff --git a/core/lib/config/src/configs/house_keeper.rs b/core/lib/config/src/configs/house_keeper.rs
@@ -4,29 +4,4 @@ use serde::Deserialize;
 #[derive(Debug, Deserialize, Clone, PartialEq)]
 pub struct HouseKeeperConfig {
     pub l1_batch_metrics_reporting_interval_ms: u64,
-    pub gpu_prover_queue_reporting_interval_ms: u64,
-    pub prover_job_retrying_interval_ms: u64,
-    pub prover_stats_reporting_interval_ms: u64,
-    pub witness_job_moving_interval_ms: u64,
-    pub witness_generator_stats_reporting_interval_ms: u64,
-    pub witness_generator_job_retrying_interval_ms: u64,
-    pub prover_db_pool_size: u32,
-    pub proof_compressor_job_retrying_interval_ms: u64,
-    pub proof_compressor_stats_reporting_interval_ms: u64,
-    pub prover_job_archiver_archiving_interval_ms: Option<u64>,
-    pub prover_job_archiver_archive_after_secs: Option<u64>,
-    pub fri_gpu_prover_archiver_archiving_interval_ms: Option<u64>,
-    pub fri_gpu_prover_archiver_archive_after_secs: Option<u64>,
-}
-
-impl HouseKeeperConfig {
-    pub fn prover_job_archiver_params(&self) -> Option<(u64, u64)> {
-        self.prover_job_archiver_archiving_interval_ms
-            .zip(self.prover_job_archiver_archive_after_secs)
-    }
-
-    pub fn fri_gpu_prover_archiver_params(&self) -> Option<(u64, u64)> {
-        self.fri_gpu_prover_archiver_archiving_interval_ms
-            .zip(self.fri_gpu_prover_archiver_archive_after_secs)
-    }
 }
diff --git a/core/lib/config/src/testonly.rs b/core/lib/config/src/testonly.rs
@@ -631,19 +631,6 @@ impl Distribution<configs::house_keeper::HouseKeeperConfig> for EncodeDist {
     fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> configs::house_keeper::HouseKeeperConfig {
         configs::house_keeper::HouseKeeperConfig {
             l1_batch_metrics_reporting_interval_ms: self.sample(rng),
-            gpu_prover_queue_reporting_interval_ms: self.sample(rng),
-            prover_job_retrying_interval_ms: self.sample(rng),
-            prover_stats_reporting_interval_ms: self.sample(rng),
-            witness_job_moving_interval_ms: self.sample(rng),
-            witness_generator_stats_reporting_interval_ms: self.sample(rng),
-            prover_db_pool_size: self.sample(rng),
-            witness_generator_job_retrying_interval_ms: self.sample(rng),
-            proof_compressor_job_retrying_interval_ms: self.sample(rng),
-            proof_compressor_stats_reporting_interval_ms: self.sample(rng),
-            prover_job_archiver_archiving_interval_ms: self.sample(rng),
-            prover_job_archiver_archive_after_secs: self.sample(rng),
-            fri_gpu_prover_archiver_archiving_interval_ms: self.sample(rng),
-            fri_gpu_prover_archiver_archive_after_secs: self.sample(rng),
         }
     }
 }

diff --git a/core/lib/dal/src/models/mod.rs b/core/lib/dal/src/models/mod.rs
@@ -15,7 +15,7 @@ pub mod storage_sync;
 pub mod storage_tee_proof;
 pub mod storage_transaction;
 pub mod storage_verification_request;
-pub mod storage_witness_job_info;
+
 #[cfg(test)]
 mod tests;