test(vm): Improve instruction-counting VM benchmark

.github/workflows/vm-perf-comparison.yml

@@ -40,6 +40,8 @@ jobs:
           echo "SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com" >> .env
           echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> .env
           echo "RUSTC_WRAPPER=sccache" >> .env
+          # Set the minimum reported instruction count difference to reduce noise
+          echo "BENCHMARK_DIFF_THRESHOLD_PERCENT=2" >> .env
 
       - name: init
         run: |
@@ -51,33 +53,48 @@ jobs:
         run: |
           ci_run zkstackup -g --local
           ci_run zkstack dev contracts --system-contracts
-          ci_run cargo bench --package vm-benchmark --bench iai | tee base-iai
-          ci_run cargo run --package vm-benchmark --release --bin instruction_counts | tee base-opcodes || touch base-opcodes
+          ci_run cargo bench --package vm-benchmark --bench instructions -- --verbose || echo "Instructions benchmark is missing"
+          ci_run cargo run --package vm-benchmark --release --bin instruction_counts | tee base-opcodes
 
       - name: checkout PR
         run: |
           git checkout --force FETCH_HEAD --recurse-submodules
 
       - name: run benchmarks on PR
         shell: bash
+        id: comparison
         run: |
           ci_run zkstackup -g --local
           ci_run zkstack dev contracts --system-contracts
-          ci_run cargo bench --package vm-benchmark --bench iai | tee pr-iai
-          ci_run cargo run --package vm-benchmark --release --bin instruction_counts | tee pr-opcodes || touch pr-opcodes
+          ci_run cargo bench --package vm-benchmark --bench instructions -- --verbose
 
+          ci_run cargo bench --package vm-benchmark --bench instructions -- --print > instructions.log 2>/dev/null
+          # Output all lines from the benchmark result starting from the "## ..." comparison header.
+          # Since the output spans multiple lines, we use a heredoc declaration.
           EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
           echo "speedup<<$EOF" >> $GITHUB_OUTPUT
-          ci_run cargo run --package vm-benchmark --release --bin compare_iai_results base-iai pr-iai base-opcodes pr-opcodes >> $GITHUB_OUTPUT
+          sed -n '/^## /,$p' instructions.log >> $GITHUB_OUTPUT
+          echo "$EOF" >> $GITHUB_OUTPUT
+
+          ci_run cargo run --package vm-benchmark --release --bin instruction_counts -- --diff base-opcodes > opcodes.log
+          echo "opcodes<<$EOF" >> $GITHUB_OUTPUT
+          sed -n '/^## /,$p' opcodes.log >> $GITHUB_OUTPUT
           echo "$EOF" >> $GITHUB_OUTPUT
-        id: comparison
 
       - name: Comment on PR
         uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6 # v2.5.0
+        if: steps.comparison.outputs.speedup != '' || steps.comparison.outputs.opcodes != ''
         with:
           message: |
-            ${{ steps.comparison.outputs.speedup == '' && '## No performance difference detected (anymore)' || '## Detected VM performance changes' }}
             ${{ steps.comparison.outputs.speedup }}
+            ${{ steps.comparison.outputs.opcodes }}
           comment_tag: vm-performance-changes
           mode: recreate
-          create_if_not_exists: ${{ steps.comparison.outputs.speedup != '' }}
+          create_if_not_exists: true
+      - name: Remove PR comment
+        uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6 # v2.5.0
+        if: steps.comparison.outputs.speedup == '' && steps.comparison.outputs.opcodes == ''
+        with:
+          comment_tag: vm-performance-changes
+          message: 'No performance difference detected (anymore)'
+          mode: delete

.github/workflows/vm-perf-to-prometheus.yml

@@ -48,5 +48,5 @@ jobs:
           ci_run cargo bench --package vm-benchmark --bench oneshot
           # Run only benches with 1,000 transactions per batch to not spend too much time
           ci_run cargo bench --package vm-benchmark --bench batch '/1000$'
-          ci_run cargo bench --package vm-benchmark --bench iai | tee iai-result
-          ci_run cargo run --package vm-benchmark --bin iai_results_to_prometheus --release < iai-result
+          ci_run cargo bench --package vm-benchmark --bench instructions -- --verbose
+          ci_run cargo bench --package vm-benchmark --bench instructions -- --print

Cargo.toml

@@ -122,6 +122,7 @@ derive_more = "1.0.0"
 envy = "0.4"
 ethabi = "18.0.0"
 flate2 = "1.0.28"
+fraction = "0.15.3"
 futures = "0.3"
 glob = "0.3"
 google-cloud-auth = "0.16.0"
@@ -131,7 +132,6 @@ hex = "0.4"
 http = "1.1"
 httpmock = "0.7.0"
 hyper = "1.3"
-iai = "0.1"
 insta = "1.29.0"
 itertools = "0.10"
 jsonrpsee = { version = "0.23", default-features = false }
@@ -190,7 +190,7 @@ tracing-opentelemetry = "0.25.0"
 time = "0.3.36"                                                               # Has to be same as used by `tracing-subscriber`
 url = "2"
 web3 = "0.19.0"
-fraction = "0.15.3"
+yab = "0.1.0"
 
 # Proc-macro
 syn = "2.0"

core/lib/multivm/src/versions/vm_fast/mod.rs

@@ -1,4 +1,4 @@
-pub use zksync_vm2::interface::Tracer;
+pub use zksync_vm2::interface;
 
 pub use self::{circuits_tracer::CircuitsTracer, vm::Vm};
 

core/lib/vm_executor/src/batch/factory.rs

@@ -37,7 +37,7 @@ pub trait BatchTracer: fmt::Debug + 'static + Send + Sealed {
     const TRACE_CALLS: bool;
     /// Tracer for the fast VM.
     #[doc(hidden)]
-    type Fast: vm_fast::Tracer + Default + 'static;
+    type Fast: vm_fast::interface::Tracer + Default + 'static;
 }
 
 impl Sealed for () {}

core/tests/vm-benchmark/Cargo.toml

@@ -21,7 +21,7 @@ tokio.workspace = true
 
 [dev-dependencies]
 assert_matches.workspace = true
-iai.workspace = true
+yab.workspace = true
 
 [[bench]]
 name = "oneshot"
@@ -32,5 +32,5 @@ name = "batch"
 harness = false
 
 [[bench]]
-name = "iai"
+name = "instructions"
 harness = false