Update from upstream

.github/release-please/manifest.json

@@ -1,5 +1,5 @@
 {
-  "core": "24.28.0",
+  "core": "24.29.0",
   "prover": "16.5.0",
   "zkstack_cli": "0.1.2"
 }

.github/workflows/build-core-template.yml

@@ -128,8 +128,9 @@ jobs:
           ci_run git config --global --add safe.directory /usr/src/zksync/contracts
           ci_run ./bin/zk || true
           ci_run run_retried curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^26.key
-      
+
       - name: Install zkstack
+        if: env.BUILD_CONTRACTS == 'true'
         run: |
           ci_run ./zkstack_cli/zkstackup/install -g --path ./zkstack_cli/zkstackup/zkstackup || true
           ci_run zkstackup -g --local
@@ -138,7 +139,7 @@ jobs:
         if: env.BUILD_CONTRACTS == 'true'
         run: |
           ci_run cp etc/tokens/{test,localhost}.json
-          ci_run zkstack dev contracts
+          ci_run zkstack dev contracts --system-contracts --l1-contracts --l2-contracts 
 
       - name: Login to Docker registries
         if: ${{ inputs.action == 'push' }}

.github/workflows/new-build-contract-verifier-template.yml

@@ -103,8 +103,13 @@ jobs:
           crate: sqlx-cli
           tag: 0.8.1
 
-      - name: Install Foundry
-        uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0
+      - name: Install foundry-zksync
+        run: |
+          mkdir ./foundry-zksync
+          curl -LO https://github.com/matter-labs/foundry-zksync/releases/download/nightly-15bec2f861b3b4c71e58f85e2b2c9dd722585aa8/foundry_nightly_linux_amd64.tar.gz
+          tar zxf foundry_nightly_linux_amd64.tar.gz -C ./foundry-zksync
+          chmod +x ./foundry-zksync/forge ./foundry-zksync/cast
+          echo "$PWD/foundry-zksync" >> $GITHUB_PATH
 
       - name: Pre-download compilers
         shell: bash
@@ -131,7 +136,7 @@ jobs:
           docker compose up -d postgres
 
       - name: Install zkstack
-        run: |  
+        run: |
           ./zkstack_cli/zkstackup/install --path ./zkstack_cli/zkstackup/zkstackup
           zkstackup --local || true
 

.github/workflows/new-build-core-template.yml

@@ -108,8 +108,13 @@ jobs:
           crate: sqlx-cli
           tag: 0.8.1
 
-      - name: Install Foundry
-        uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0
+      - name: Install foundry-zksync
+        run: |
+          mkdir ./foundry-zksync
+          curl -LO https://github.com/matter-labs/foundry-zksync/releases/download/nightly-15bec2f861b3b4c71e58f85e2b2c9dd722585aa8/foundry_nightly_linux_amd64.tar.gz
+          tar zxf foundry_nightly_linux_amd64.tar.gz -C ./foundry-zksync
+          chmod +x ./foundry-zksync/forge ./foundry-zksync/cast
+          echo "$PWD/foundry-zksync" >> $GITHUB_PATH
 
       - name: Pre-download compilers
         shell: bash
@@ -136,10 +141,10 @@ jobs:
           docker compose up -d postgres
 
       - name: Install zkstack
-        run: |  
+        run: |
           ./zkstack_cli/zkstackup/install --path ./zkstack_cli/zkstackup/zkstackup
           zkstackup --local || true
-  
+
       - name: build contracts
         shell: bash
         run: |

.github/workflows/vm-perf-to-prometheus.yml

@@ -28,17 +28,17 @@ jobs:
           echo "RUSTC_WRAPPER=sccache" >> .env
           echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
           echo $(pwd)/bin >> $GITHUB_PATH
-          
-      - name: Install zkstack
-        run: |
-          ci_run ./zkstack_cli/zkstackup/install -g --path ./zkstack_cli/zkstackup/zkstackup || true
-          ci_run zkstackup -g --local
 
       - name: init
         run: |
           run_retried docker compose pull zk
           docker compose up -d zk
 
+      - name: Install zkstack
+        run: |
+          ci_run ./zkstack_cli/zkstackup/install -g --path ./zkstack_cli/zkstackup/zkstackup || true
+          ci_run zkstackup -g --local
+
       - name: build contracts
         run: |
           ci_run zkstack dev contracts

core/CHANGELOG.md

@@ -1,5 +1,43 @@
 # Changelog
 
+## [24.29.0](https://github.com/matter-labs/zksync-era/compare/core-v24.28.0...core-v24.29.0) (2024-10-14)
+
+
+### Features
+
+* Add  initial version prover_autoscaler ([#2993](https://github.com/matter-labs/zksync-era/issues/2993)) ([ebf9604](https://github.com/matter-labs/zksync-era/commit/ebf9604c5ab2a1cae1ffd2f9c922f35a1d0ad876))
+* add metric to track current cbt ratio ([#3020](https://github.com/matter-labs/zksync-era/issues/3020)) ([3fd2fb1](https://github.com/matter-labs/zksync-era/commit/3fd2fb14e7283c6858731e162522e70051a8e162))
+* **configs:** Add port parameter to ConsensusConfig ([#2986](https://github.com/matter-labs/zksync-era/issues/2986)) ([25112df](https://github.com/matter-labs/zksync-era/commit/25112df39d052f083bc45964f0298b3af5842cac))
+* **configs:** Add port parameter to ConsensusConfig ([#3051](https://github.com/matter-labs/zksync-era/issues/3051)) ([038c397](https://github.com/matter-labs/zksync-era/commit/038c397ce842601da5109c460b09dbf9d51cf2fc))
+* **consensus:** smooth transition to p2p syncing (BFT-515) ([#3075](https://github.com/matter-labs/zksync-era/issues/3075)) ([5d339b4](https://github.com/matter-labs/zksync-era/commit/5d339b46fee66bc3a45493586626d318380680dd))
+* **consensus:** Support for syncing blocks before consensus genesis over p2p network ([#3040](https://github.com/matter-labs/zksync-era/issues/3040)) ([d3edc3d](https://github.com/matter-labs/zksync-era/commit/d3edc3d817c151ed00d4fa822fdae0a746e33356))
+* **en:** periodically fetch bridge addresses ([#2949](https://github.com/matter-labs/zksync-era/issues/2949)) ([e984bfb](https://github.com/matter-labs/zksync-era/commit/e984bfb8a243bc746549ab9347dc0a367fe02790))
+* **eth-sender:** add time_in_mempool_cap config ([#3018](https://github.com/matter-labs/zksync-era/issues/3018)) ([f6d86bd](https://github.com/matter-labs/zksync-era/commit/f6d86bd7935a1cdbb528b13437424031fda3cb8e))
+* **eth-watch:** catch another reth error ([#3026](https://github.com/matter-labs/zksync-era/issues/3026)) ([4640c42](https://github.com/matter-labs/zksync-era/commit/4640c4233af46c97f207d2dbce5fedd1bcb66c43))
+* Handle new yul compilation flow ([#3038](https://github.com/matter-labs/zksync-era/issues/3038)) ([4035361](https://github.com/matter-labs/zksync-era/commit/40353616f278800dc80fcbe5f2a6483019033b20))
+* **state-keeper:** pre-insert unsealed L1 batches ([#2846](https://github.com/matter-labs/zksync-era/issues/2846)) ([e5b5a3b](https://github.com/matter-labs/zksync-era/commit/e5b5a3b7b62e8d4035fe89c2a287bf3606d17bc5))
+* **vm:** EVM emulator support – base ([#2979](https://github.com/matter-labs/zksync-era/issues/2979)) ([deafa46](https://github.com/matter-labs/zksync-era/commit/deafa460715334a77edf9fe8aa76fa90029342c4))
+* **zk_toolbox:** added support for setting attester committee defined in a separate file ([#2992](https://github.com/matter-labs/zksync-era/issues/2992)) ([6105514](https://github.com/matter-labs/zksync-era/commit/610551427d5ab129f91e69b5efb318da917457d7))
+* **zk_toolbox:** Redesign zk_toolbox commands ([#3003](https://github.com/matter-labs/zksync-era/issues/3003)) ([114834f](https://github.com/matter-labs/zksync-era/commit/114834f357421c62d596a1954fac8ce615cfde49))
+* **zktoolbox:** added checking the contract owner in set-attester-committee command ([#3061](https://github.com/matter-labs/zksync-era/issues/3061)) ([9b0a606](https://github.com/matter-labs/zksync-era/commit/9b0a6067923c5276f560f3abccedc4e6a5167dda))
+
+
+### Bug Fixes
+
+* **api:** Accept integer block count in `eth_feeHistory` ([#3077](https://github.com/matter-labs/zksync-era/issues/3077)) ([4d527d4](https://github.com/matter-labs/zksync-era/commit/4d527d4b44b6b083e2a813d48c79d8021ea6f843))
+* **api:** Adapt `eth_getCode` to EVM emulator ([#3073](https://github.com/matter-labs/zksync-era/issues/3073)) ([15fe5a6](https://github.com/matter-labs/zksync-era/commit/15fe5a62f03cd103afd7fa5eb03e27db25686ba9))
+* bincode deserialization for VM run data ([#3044](https://github.com/matter-labs/zksync-era/issues/3044)) ([b0ec79f](https://github.com/matter-labs/zksync-era/commit/b0ec79fcb7fa120f095d987f53c67fdab92e2c79))
+* bincode deserialize for WitnessInputData ([#3055](https://github.com/matter-labs/zksync-era/issues/3055)) ([91d0595](https://github.com/matter-labs/zksync-era/commit/91d0595631cc5f5bffc42a4b04d5015d2be659b1))
+* **external-node:** make fetcher rely on unsealed batches  ([#3088](https://github.com/matter-labs/zksync-era/issues/3088)) ([bb5d147](https://github.com/matter-labs/zksync-era/commit/bb5d1470d5e1e8e69d9b79c60284ea8adaee4038))
+* **state-keeper:** ensure unsealed batch is present during IO init ([#3071](https://github.com/matter-labs/zksync-era/issues/3071)) ([bdeb411](https://github.com/matter-labs/zksync-era/commit/bdeb411c593ac3d5e16158e64c4210bb00edcb0c))
+* **vm:** Check protocol version for fast VM ([#3080](https://github.com/matter-labs/zksync-era/issues/3080)) ([a089f3f](https://github.com/matter-labs/zksync-era/commit/a089f3feb916ccc9007d9c32ec909db694b7d9f4))
+* **vm:** Prepare new VM for use in API server and fix divergences ([#2994](https://github.com/matter-labs/zksync-era/issues/2994)) ([741b77e](https://github.com/matter-labs/zksync-era/commit/741b77e080f75c6a93d3ee779b1c9ce4297618f9))
+
+
+### Reverts
+
+* **configs:** Add port parameter to ConsensusConfig ([#2986](https://github.com/matter-labs/zksync-era/issues/2986)) ([#3046](https://github.com/matter-labs/zksync-era/issues/3046)) ([abe35bf](https://github.com/matter-labs/zksync-era/commit/abe35bf7aea1120b77fdbd413d927e45da48d26c))
+
 ## [24.28.0](https://github.com/matter-labs/zksync-era/compare/core-v24.27.0...core-v24.28.0) (2024-10-02)
 
 

core/bin/external_node/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "zksync_external_node"
 description = "Non-validator ZKsync node"
-version = "24.28.0" # x-release-please-version
+version = "24.29.0" # x-release-please-version
 edition.workspace = true
 authors.workspace = true
 homepage.workspace = true

core/bin/external_node/src/node_builder.rs

@@ -378,6 +378,7 @@ impl ExternalNodeBuilder {
             factory_deps_cache_size: self.config.optional.factory_deps_cache_size() as u64,
             initial_writes_cache_size: self.config.optional.initial_writes_cache_size() as u64,
             latest_values_cache_size: self.config.optional.latest_values_cache_size() as u64,
+            latest_values_max_block_lag: 20, // reasonable default
         };
         let max_vm_concurrency = self.config.optional.vm_concurrency_limit;
         let tx_sender_layer = TxSenderLayer::new(

core/bin/system-constants-generator/src/utils.rs

@@ -3,7 +3,7 @@ use std::{cell::RefCell, rc::Rc};
 use once_cell::sync::Lazy;
 use zksync_contracts::{
     load_sys_contract, read_bootloader_code, read_bytecode_from_path, read_sys_contract_bytecode,
-    BaseSystemContracts, ContractLanguage, SystemContractCode,
+    read_zbin_bytecode, BaseSystemContracts, ContractLanguage, SystemContractCode,
 };
 use zksync_multivm::{
     interface::{
@@ -171,9 +171,16 @@ pub(super) fn get_l1_txs(number_of_txs: usize) -> (Vec<Transaction>, Vec<Transac
 }
 
 fn read_bootloader_test_code(test: &str) -> Vec<u8> {
-    read_bytecode_from_path(format!(
+    if let Some(contract) = read_bytecode_from_path(format!(
         "contracts/system-contracts/zkout/{test}.yul/contracts-preprocessed/bootloader/{test}.yul.json",
-    ))
+    )){
+        contract
+    } else  {
+        read_zbin_bytecode(format!(
+            "contracts/system-contracts/bootloader/tests/artifacts/{}.yul.zbin",
+            test
+        ))
+    }
 }
 
 fn default_l1_batch() -> L1BatchEnv {

core/bin/zksync_server/src/node_builder.rs

@@ -19,9 +19,7 @@ use zksync_node_framework::{
     implementations::layers::{
         base_token::{
             base_token_ratio_persister::BaseTokenRatioPersisterLayer,
-            base_token_ratio_provider::BaseTokenRatioProviderLayer,
-            coingecko_client::CoingeckoClientLayer, forced_price_client::ForcedPriceClientLayer,
-            no_op_external_price_api_client::NoOpExternalPriceApiClientLayer,
+            base_token_ratio_provider::BaseTokenRatioProviderLayer, ExternalPriceApiLayer,
         },
         circuit_breaker_checker::CircuitBreakerCheckerLayer,
         commitment_generator::CommitmentGeneratorLayer,
@@ -307,6 +305,7 @@ impl MainNodeBuilder {
             factory_deps_cache_size: rpc_config.factory_deps_cache_size() as u64,
             initial_writes_cache_size: rpc_config.initial_writes_cache_size() as u64,
             latest_values_cache_size: rpc_config.latest_values_cache_size() as u64,
+            latest_values_max_block_lag: rpc_config.latest_values_max_block_lag(),
         };
 
         // On main node we always use master pool sink.
@@ -557,24 +556,8 @@ impl MainNodeBuilder {
 
     fn add_external_api_client_layer(mut self) -> anyhow::Result<Self> {
         let config = try_load_config!(self.configs.external_price_api_client_config);
-        match config.source.as_str() {
-            CoingeckoClientLayer::CLIENT_NAME => {
-                self.node.add_layer(CoingeckoClientLayer::new(config));
-            }
-            NoOpExternalPriceApiClientLayer::CLIENT_NAME => {
-                self.node.add_layer(NoOpExternalPriceApiClientLayer);
-            }
-            ForcedPriceClientLayer::CLIENT_NAME => {
-                self.node.add_layer(ForcedPriceClientLayer::new(config));
-            }
-            _ => {
-                anyhow::bail!(
-                    "Unknown external price API client source: {}",
-                    config.source
-                );
-            }
-        }
-
+        self.node
+            .add_layer(ExternalPriceApiLayer::try_from(config)?);
         Ok(self)
     }
 

core/lib/basic_types/src/protocol_version.rs

@@ -68,15 +68,16 @@ pub enum ProtocolVersionId {
     Version23,
     Version24,
     Version25,
+    Version26,
 }
 
 impl ProtocolVersionId {
     pub const fn latest() -> Self {
-        Self::Version24
+        Self::Version25
     }
 
     pub const fn next() -> Self {
-        Self::Version25
+        Self::Version26
     }
 
     pub fn try_from_packed_semver(packed_semver: U256) -> Result<Self, String> {
@@ -120,6 +121,7 @@ impl ProtocolVersionId {
             ProtocolVersionId::Version23 => VmVersion::Vm1_5_0SmallBootloaderMemory,
             ProtocolVersionId::Version24 => VmVersion::Vm1_5_0IncreasedBootloaderMemory,
             ProtocolVersionId::Version25 => VmVersion::Vm1_5_0IncreasedBootloaderMemory,
+            ProtocolVersionId::Version26 => VmVersion::Vm1_5_0IncreasedBootloaderMemory,
         }
     }
 
@@ -275,6 +277,7 @@ impl From<ProtocolVersionId> for VmVersion {
             ProtocolVersionId::Version23 => VmVersion::Vm1_5_0SmallBootloaderMemory,
             ProtocolVersionId::Version24 => VmVersion::Vm1_5_0IncreasedBootloaderMemory,
             ProtocolVersionId::Version25 => VmVersion::Vm1_5_0IncreasedBootloaderMemory,
+            ProtocolVersionId::Version26 => VmVersion::Vm1_5_0IncreasedBootloaderMemory,
         }
     }
 }

core/lib/config/src/configs/api.rs

@@ -189,6 +189,10 @@ pub struct Web3JsonRpcConfig {
     /// Latest values cache size in MiBs. The default value is 128 MiB. If set to 0, the latest
     /// values cache will be disabled.
     pub latest_values_cache_size_mb: Option<usize>,
+    /// Maximum lag in the number of blocks for the latest values cache after which the cache is reset. Greater values
+    /// lead to increased the cache update latency, i.e., less storage queries being processed by the cache. OTOH, smaller values
+    /// can lead to spurious resets when Postgres lags for whatever reason (e.g., when sealing L1 batches).
+    pub latest_values_max_block_lag: Option<NonZeroU32>,
     /// Limit for fee history block range.
     pub fee_history_limit: Option<u64>,
     /// Maximum number of requests in a single batch JSON RPC request. Default is 500.
@@ -243,20 +247,21 @@ impl Web3JsonRpcConfig {
             estimate_gas_acceptable_overestimation: 1000,
             estimate_gas_optimize_search: false,
             max_tx_size: 1000000,
-            vm_execution_cache_misses_limit: Default::default(),
-            vm_concurrency_limit: Default::default(),
-            factory_deps_cache_size_mb: Default::default(),
-            initial_writes_cache_size_mb: Default::default(),
-            latest_values_cache_size_mb: Default::default(),
-            fee_history_limit: Default::default(),
-            max_batch_request_size: Default::default(),
-            max_response_body_size_mb: Default::default(),
+            vm_execution_cache_misses_limit: None,
+            vm_concurrency_limit: None,
+            factory_deps_cache_size_mb: None,
+            initial_writes_cache_size_mb: None,
+            latest_values_cache_size_mb: None,
+            latest_values_max_block_lag: None,
+            fee_history_limit: None,
+            max_batch_request_size: None,
+            max_response_body_size_mb: None,
             max_response_body_size_overrides_mb: MaxResponseSizeOverrides::empty(),
-            websocket_requests_per_minute_limit: Default::default(),
-            mempool_cache_update_interval: Default::default(),
-            mempool_cache_size: Default::default(),
+            websocket_requests_per_minute_limit: None,
+            mempool_cache_update_interval: None,
+            mempool_cache_size: None,
             tree_api_url: None,
-            whitelisted_tokens_for_aa: Default::default(),
+            whitelisted_tokens_for_aa: vec![],
             api_namespaces: None,
             extended_api_tracing: false,
         }
@@ -308,6 +313,11 @@ impl Web3JsonRpcConfig {
         self.latest_values_cache_size_mb.unwrap_or(128) * super::BYTES_IN_MEGABYTE
     }
 
+    /// Returns the maximum lag in the number of blocks for the latest values cache.
+    pub fn latest_values_max_block_lag(&self) -> u32 {
+        self.latest_values_max_block_lag.map_or(20, NonZeroU32::get)
+    }
+
     pub fn fee_history_limit(&self) -> u64 {
         self.fee_history_limit.unwrap_or(1024)
     }

core/lib/config/src/testonly.rs

@@ -86,6 +86,7 @@ impl Distribution<configs::api::Web3JsonRpcConfig> for EncodeDist {
             factory_deps_cache_size_mb: self.sample(rng),
             initial_writes_cache_size_mb: self.sample(rng),
             latest_values_cache_size_mb: self.sample(rng),
+            latest_values_max_block_lag: self.sample(rng),
             fee_history_limit: self.sample(rng),
             max_batch_request_size: self.sample(rng),
             max_response_body_size_mb: self.sample(rng),