Merge branch 'feat/migrate_compression' into feat/update-prover-migra…

…te-compression

aggregator/Cargo.toml

@@ -45,5 +45,6 @@ zstd-encoder = { package = "encoder", git = "https://github.com/scroll-tech/da-c
 csv = "1.1"
 
 [features]
-default = ["revm-precompile/c-kzg"]
+default = ["revm-precompile/c-kzg", "halo2_proofs/circuit-params"]
+display = []
 print-trace = ["ark-std/print-trace"]

aggregator/src/aggregation/decoder/seq_exec.rs

@@ -1128,7 +1128,6 @@ mod tests {
         type Config = SeqExecMockConfig;
         type FloorPlanner = SimpleFloorPlanner;
         type Params = ();
-
         fn without_witnesses(&self) -> Self {
             unimplemented!()
         }

aggregator/src/aggregation/decoder/tables/fixed/fse_table_transition.rs

@@ -2,16 +2,7 @@ use halo2_proofs::{circuit::Value, halo2curves::bn256::Fr};
 
 use super::{FixedLookupTag, FixedLookupValues};
 
-pub struct RomFseTableTransition {
-    /// The block index on the previous FSE table.
-    pub block_idx_prev: u64,
-    /// The block index on the current FSE table.
-    pub block_idx_curr: u64,
-    /// The FSE table previously decoded.
-    pub table_kind_prev: u64,
-    /// The FSE table currently decoded.
-    pub table_kind_curr: u64,
-}
+pub struct RomFseTableTransition;
 
 impl FixedLookupValues for RomFseTableTransition {
     fn values() -> Vec<[Value<Fr>; 7]> {

aggregator/src/aggregation/decoder/tables/fixed/seq_data_interleaved_order.rs

@@ -5,16 +5,7 @@ use crate::aggregation::decoder::{
     witgen::FseTableKind,
 };
 
-pub struct RomSeqDataInterleavedOrder {
-    /// FSE table used in the previous bitstring.
-    pub table_kind_prev: FseTableKind,
-    /// FSE table used in the current bitstring.
-    pub table_kind_curr: FseTableKind,
-    /// Boolean flag to indicate whether we are initialising the FSE state.
-    pub is_init_state: bool,
-    /// Boolean flag to indicate whether we are updating the FSE state.
-    pub is_update_state: bool,
-}
+pub struct RomSeqDataInterleavedOrder;
 
 impl FixedLookupValues for RomSeqDataInterleavedOrder {
     fn values() -> Vec<[Value<Fr>; 7]> {

aggregator/src/aggregation/decoder/tables/fixed/seq_tag_order.rs

@@ -28,22 +28,7 @@ use super::FixedLookupValues;
 ///     - SequenceHeader > FseCode > SequenceData (MOT)
 /// - (0, 0, 1):
 ///     - SequenceHeader > FseCode > SequenceData (MLT)
-pub struct RomSeqTagOrder {
-    /// Boolean flag to mark if LLT is Fse_Compressed_Mode or Predefined_Mode.
-    pub cmode_llt: bool,
-    /// Boolean flag to mark if MOT is Fse_Compressed_Mode or Predefined_Mode.
-    pub cmode_mot: bool,
-    /// Boolean flag to mark if MLT is Fse_Compressed_Mode or Predefined_Mode.
-    pub cmode_mlt: bool,
-    /// Tag that was handled before the current tag.
-    pub tag_prev: ZstdTag,
-    /// Tag currently being handled.
-    pub tag_curr: ZstdTag,
-    /// Tag that will be handled after the current tag.
-    pub tag_next: ZstdTag,
-    /// The FSE table that we expect with the current tag.
-    pub fse_table: FseTableKind,
-}
+pub struct RomSeqTagOrder;
 
 impl FixedLookupValues for RomSeqTagOrder {
     fn values() -> Vec<[Value<Fr>; 7]> {

aggregator/src/aggregation/decoder/tables/fixed/tag_transition.rs

@@ -4,18 +4,7 @@ use crate::aggregation::decoder::{tables::fixed::FixedLookupTag, witgen::ZstdTag
 
 use super::FixedLookupValues;
 
-pub struct RomTagTransition {
-    /// The current tag.
-    pub tag: ZstdTag,
-    /// The tag that will be processed after the current tag is finished processing.
-    pub tag_next: ZstdTag,
-    /// The maximum number of bytes that are needed to represent the current tag.
-    pub max_len: u64,
-    /// Whether this tag is processed from back-to-front or not.
-    pub is_reverse: bool,
-    /// Whether this tag belongs to a ``block`` in zstd or not.
-    pub is_block: bool,
-}
+pub struct RomTagTransition;
 
 impl FixedLookupValues for RomTagTransition {
     fn values() -> Vec<[Value<Fr>; 7]> {

aggregator/src/aggregation/decoder/tables/seqinst_table.rs

@@ -1004,7 +1004,6 @@ mod tests {
         type Config = SeqInstTable<Fr>;
         type FloorPlanner = SimpleFloorPlanner;
         type Params = ();
-
         fn without_witnesses(&self) -> Self {
             unimplemented!()
         }

aggregator/src/aggregation/decoder/witgen.rs

@@ -295,7 +295,6 @@ pub struct LiteralsBlockResult<F> {
     pub offset: usize,
     pub witness_rows: Vec<ZstdWitnessRow<F>>,
     pub literals: Vec<u64>,
-    pub regen_size: usize,
 }
 
 #[allow(clippy::too_many_arguments)]
@@ -318,7 +317,6 @@ fn process_block_zstd<F: Field>(
         offset: byte_offset,
         witness_rows: rows,
         regen_size,
-        compressed_size: _,
     } = process_block_zstd_literals_header::<F>(src, block_idx, byte_offset, last_row, randomness);
 
     witness_rows.extend_from_slice(&rows);
@@ -327,7 +325,6 @@ fn process_block_zstd<F: Field>(
         offset: byte_offset,
         witness_rows: rows,
         literals,
-        regen_size: _,
     } = {
         let last_row = rows.last().cloned().unwrap();
         let multiplier =
@@ -376,7 +373,6 @@ fn process_block_zstd<F: Field>(
                 })
                 .collect::<Vec<_>>(),
             literals: literals.iter().map(|b| *b as u64).collect::<Vec<u64>>(),
-            regen_size,
         }
     };
 
@@ -1606,7 +1602,6 @@ pub struct LiteralsHeaderProcessingResult<F> {
     pub offset: usize,
     pub witness_rows: Vec<ZstdWitnessRow<F>>,
     pub regen_size: usize,
-    pub compressed_size: usize,
 }
 
 fn process_block_zstd_literals_header<F: Field>(
@@ -1626,12 +1621,11 @@ fn process_block_zstd_literals_header<F: Field>(
     let literals_block_type = BlockType::from(lh_bytes[0] & 0x3);
     let size_format = (lh_bytes[0] >> 2) & 3;
 
-    let [n_bits_fmt, n_bits_regen, n_bits_compressed, _n_streams, n_bytes_header, _branch]: [usize;
-        6] = match literals_block_type {
+    let [n_bits_fmt, n_bits_regen, n_bytes_header]: [usize; 3] = match literals_block_type {
         BlockType::RawBlock => match size_format {
-            0b00 | 0b10 => [1, 5, 0, 1, 1, 0],
-            0b01 => [2, 12, 0, 1, 2, 1],
-            0b11 => [2, 20, 0, 1, 3, 2],
+            0b00 | 0b10 => [1, 5, 1],
+            0b01 => [2, 12, 2],
+            0b11 => [2, 20, 3],
             _ => unreachable!("size_format out of bound"),
         },
         _ => unreachable!("BlockType::* unexpected. Must be raw bytes for literals."),
@@ -1644,8 +1638,6 @@ fn process_block_zstd_literals_header<F: Field>(
     })[(2 + n_bits_fmt)..(n_bytes_header * N_BITS_PER_BYTE)];
 
     let regen_size = le_bits_to_value(&sizing_bits[0..n_bits_regen]) as usize;
-    let compressed_size =
-        le_bits_to_value(&sizing_bits[n_bits_regen..(n_bits_regen + n_bits_compressed)]) as usize;
     let tag_next = match literals_block_type {
         BlockType::RawBlock => ZstdTag::ZstdBlockLiteralsRawBytes,
         _ => unreachable!("BlockType::* unexpected. Must be raw bytes for literals."),
@@ -1698,7 +1690,6 @@ fn process_block_zstd_literals_header<F: Field>(
             })
             .collect::<Vec<_>>(),
         regen_size,
-        compressed_size,
     }
 }
 

aggregator/src/aggregation/decoder/witgen/types.rs

@@ -187,19 +187,23 @@ pub enum FseTableKind {
 
 impl_expr!(FseTableKind);
 
-impl ToString for ZstdTag {
-    fn to_string(&self) -> String {
-        String::from(match self {
-            Self::Null => "null",
-            Self::FrameHeaderDescriptor => "FrameHeaderDescriptor",
-            Self::FrameContentSize => "FrameContentSize",
-            Self::BlockHeader => "BlockHeader",
-            Self::ZstdBlockLiteralsHeader => "ZstdBlockLiteralsHeader",
-            Self::ZstdBlockLiteralsRawBytes => "ZstdBlockLiteralsRawBytes",
-            Self::ZstdBlockSequenceHeader => "ZstdBlockSequenceHeader",
-            Self::ZstdBlockSequenceFseCode => "ZstdBlockSequenceFseCode",
-            Self::ZstdBlockSequenceData => "ZstdBlockSequenceData",
-        })
+impl std::fmt::Display for ZstdTag {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{}",
+            match self {
+                Self::Null => "null",
+                Self::FrameHeaderDescriptor => "FrameHeaderDescriptor",
+                Self::FrameContentSize => "FrameContentSize",
+                Self::BlockHeader => "BlockHeader",
+                Self::ZstdBlockLiteralsHeader => "ZstdBlockLiteralsHeader",
+                Self::ZstdBlockLiteralsRawBytes => "ZstdBlockLiteralsRawBytes",
+                Self::ZstdBlockSequenceHeader => "ZstdBlockSequenceHeader",
+                Self::ZstdBlockSequenceFseCode => "ZstdBlockSequenceFseCode",
+                Self::ZstdBlockSequenceData => "ZstdBlockSequenceData",
+            }
+        )
     }
 }
 
@@ -523,17 +527,6 @@ impl SequenceFixedStateActionTable {
     }
 }
 
-/// Data for the FSE table's witness values.
-#[derive(Clone, Debug)]
-pub struct FseTableData {
-    /// The byte offset in the frame at which the FSE table is described.
-    pub byte_offset: u64,
-    /// The FSE table's size, i.e. 1 << AL (accuracy log).
-    pub table_size: u64,
-    /// Represent the states, symbols, and so on of this FSE table.
-    pub rows: Vec<FseTableRow>,
-}
-
 /// Auxiliary data accompanying the FSE table's witness values.
 #[derive(Clone, Debug)]
 pub struct FseAuxiliaryTableData {
@@ -787,7 +780,7 @@ impl FseAuxiliaryTableData {
             let mut count = 0;
             let mut states_with_skipped: Vec<(u64, bool)> = Vec::with_capacity(N);
             while count < N {
-                if allocated_states.get(&state).is_some() {
+                if allocated_states.contains_key(&state) {
                     // if state has been pre-allocated to some symbol with prob=-1.
                     states_with_skipped.push((state, true));
                 } else {
@@ -949,8 +942,7 @@ mod tests {
                 (0x1e, 0x0c, 2),
             ]
             .iter()
-            .enumerate()
-            .map(|(_i, &(state, baseline, num_bits))| FseTableRow {
+            .map(|&(state, baseline, num_bits)| FseTableRow {
                 state,
                 symbol: 1,
                 baseline,

aggregator/src/aggregation/rlc/gates.rs

@@ -1,8 +1,7 @@
 use ethers_core::utils::keccak256;
 use halo2_proofs::{
-    arithmetic::Field,
     circuit::{AssignedCell, Cell, Region, RegionIndex, Value},
-    halo2curves::bn256::Fr,
+    halo2curves::{bn256::Fr, ff::Field},
     plonk::Error,
 };
 use zkevm_circuits::util::Challenges;

aggregator/src/lib.rs

@@ -1,4 +1,4 @@
-#![feature(lazy_cell)]
+#![allow(clippy::doc_lazy_continuation)]
 /// proof aggregation
 mod aggregation;
 /// This module implements `Batch` related data types.

aggregator/src/recursion/circuit.rs

@@ -529,7 +529,7 @@ impl<ST: StateTransition> Circuit<Fr> for RecursionCircuit<ST> {
                 #[cfg(feature = "display")]
                 dbg!(ctx.total_advice);
                 #[cfg(feature = "display")]
-                println!("Advice columns used: {}", ctx.advice_alloc[0][0].0 + 1);
+                println!("Advice columns used: {:?}", ctx.advice_alloc[0]);
 
                 // Return the computed instance cells for this Recursion Circuit.
                 Ok([lhs.x(), lhs.y(), rhs.x(), rhs.y()]

aggregator/src/tests/blob.rs

@@ -59,7 +59,6 @@ impl Circuit<Fr> for BlobCircuit {
     type Config = BlobConfig;
     type FloorPlanner = SimpleFloorPlanner;
     type Params = ();
-
     fn without_witnesses(&self) -> Self {
         unimplemented!()
     }

aggregator/src/tests/rlc/gates.rs

@@ -26,7 +26,6 @@ impl Circuit<Fr> for ArithTestCircuit {
     type Config = RlcConfig;
     type FloorPlanner = SimpleFloorPlanner;
     type Params = ();
-
     fn without_witnesses(&self) -> Self {
         Self::default()
     }

bus-mapping/src/circuit_input_builder.rs

@@ -147,10 +147,11 @@ impl Default for CircuitsParams {
 /// steps:
 ///
 /// 1. Take a [`eth_types::Block`] to build the circuit input associated with
-/// the block. 2. For each [`eth_types::Transaction`] in the block, take the
-/// [`eth_types::GethExecTrace`] to build the circuit input associated with
-/// each transaction, and the bus-mapping operations associated with each
-/// [`eth_types::GethExecStep`] in the [`eth_types::GethExecTrace`].
+/// the block.
+/// 2. For each [`eth_types::Transaction`] in the block, take the [`eth_types::GethExecTrace`]
+/// to build the circuit input associated with each transaction,
+/// and the bus-mapping operations associated with each [`eth_types::GethExecStep`]
+/// in the [`eth_types::GethExecTrace`].
 ///
 /// The generated bus-mapping operations are:
 /// [`StackOp`](crate::operation::StackOp)s,

bus-mapping/src/circuit_input_builder/builder_client.rs

@@ -1,6 +1,4 @@
 use eth_types::{
-    constants::SCROLL_COINBASE,
-    geth_types::{self, Account, BlockConstants},
     state_db::{self, CodeDB, StateDB},
     utils::hash_code_keccak,
     Address, EthBlock, GethExecTrace, ToWord, Word, H256, KECCAK_CODE_HASH_EMPTY,
@@ -11,7 +9,6 @@ use hex::decode_to_slice;
 use super::{AccessSet, Block, Blocks, CircuitInputBuilder, CircuitsParams};
 use crate::{error::Error, rpc::GethClient};
 
-use std::str::FromStr;
 use std::{collections::HashMap, iter};
 
 /// Struct that wraps a GethClient and contains methods to perform all the steps
@@ -461,6 +458,8 @@ impl<P: JsonRpcClient> BuilderClient<P> {
         geth_traces: impl Iterator<Item = &GethExecTrace>,
         complete_prestate: bool,
     ) -> Result<external_tracer::TraceConfig, Error> {
+        use std::str::FromStr;
+
         let (proofs, codes) = self.get_pre_state(geth_traces)?;
         let proofs = if complete_prestate {
             self.complete_prestate(eth_block, proofs).await?
@@ -470,14 +469,14 @@ impl<P: JsonRpcClient> BuilderClient<P> {
 
         // We will not need to regen pk each time if we use same coinbase.
         //let coinbase =  eth_block.author.unwrap();
-        let coinbase = Address::from_str(SCROLL_COINBASE).unwrap();
+        let coinbase = Address::from_str(eth_types::constants::SCROLL_COINBASE).unwrap();
         //let difficulty = eth_block.difficulty;
         let difficulty = Word::zero();
 
         Ok(external_tracer::TraceConfig {
             chain_id: self.chain_id,
             history_hashes: vec![eth_block.parent_hash.to_word()],
-            block_constants: BlockConstants {
+            block_constants: eth_types::geth_types::BlockConstants {
                 coinbase,
                 timestamp: eth_block.timestamp,
                 number: eth_block.number.unwrap(),
@@ -488,7 +487,7 @@ impl<P: JsonRpcClient> BuilderClient<P> {
             accounts: proofs
                 .into_iter()
                 .map(|proof| {
-                    let acc = Account {
+                    let acc = eth_types::geth_types::Account {
                         address: proof.address,
                         nonce: proof.nonce,
                         balance: proof.balance,
@@ -509,7 +508,7 @@ impl<P: JsonRpcClient> BuilderClient<P> {
             transactions: eth_block
                 .transactions
                 .iter()
-                .map(geth_types::Transaction::from)
+                .map(eth_types::geth_types::Transaction::from)
                 .collect(),
             logger_config: Default::default(),
             chain_config: None,

bus-mapping/src/lib.rs

@@ -208,6 +208,7 @@
 #![cfg_attr(docsrs, feature(doc_cfg))]
 // Temporary until we have more of the crate implemented.
 #![allow(dead_code)]
+#![allow(clippy::doc_lazy_continuation)]
 // We want to have UPPERCASE idents sometimes.
 #![allow(non_snake_case)]
 // Catch documentation errors caused by code changes.
@@ -218,7 +219,6 @@
 #![allow(clippy::result_large_err)] // it's large, but what can we do?
 #![allow(clippy::collapsible_else_if)]
 #![allow(incomplete_features)]
-#![feature(lazy_cell)]
 #![feature(adt_const_params)]
 
 extern crate alloc;