Upgrade to release v0.2.0 (axiom-crypto#11)

* Initial commit * chore: fix Cargo with tagged versions of Axiom repos * feat: add `AxiomV0StoragePf` snark verifier Yul code * Release 0.2.0 (axiom-crypto#6) * feat: add batch query circuits Also add withdrawals root to block_header * fix: remove test_mpt_empty_root since empty proof is not supported We currently do not allow MPT proofs of length 0. * feat: change `EthScheduler` to use general RPC provider url * feat: set ACCOUNT_PROOF_MAX_DEPTH = 10, STORAGE_PROOF_MAX_DEPTH = 9 * chore: remove halo2-lib git submodule Also remove halo2-lib from workspace * feat: fix versions and tags for dependencies * feat: add github CI * fix: add missing env variable for RLP test * fix(tests): empty range for keccak var len = 0 * fix(scheduler): final assembly when slot is empty (axiom-crypto#7) * fix(scheduler): row consistency task default when empty slot * fix: scheduler wrong when non-empty account, empty slot * fix(scheduler): block/account response needs to be zero for empty row consistency circuit expects block/account keccak response private input to be zero if the account (resp. slot) is empty. Scheduler did not handle this. * feat: add tests for scheduler with empty account/slots Final assembly task with: * non-empty accounts, empty slots * empty accounts, empty slots * feat(batch_query): update config files * chore: clippy --fix * fix(scheduler): use simpler resize strategy for `block_task` Instead of trying to clone `MultiBlockCircuit` input in chunks, we just extend by zero to max size. This matches what is done by the SDK. * feat: add initial config files for historical mainnet * update(block_headers): config and yul files 4b454602fc23415a0e825498a1f9e451adc4cbe2aac9e7f824f6d41c6c1702d3 data/headers/mainnet_10_7.pk 41bb6cf9f4d1c4d7eb28d6329356ff5ae20638047fafe04ba05c79f1583898a9 data/headers/mainnet_10_7_final.pk a6242c0908d1286b6328297775f1a961488123fc1165c05aacd4ecf784c4afc1 data/headers/mainnet_10_7_for_evm_0.pk d9eccff67ae3965605818ec03db0bd956de3cb844009a66aad38396cc8d25ea6 data/headers/mainnet_10_7_for_evm_1.pk 87de3424dc6243f81f21cc8b3ca3d3a8b92bc4c1ac3daf4a2f5f8e770dac9bb2 data/headers/mainnet_11_7.pk 3021696fe800eaf1acc1dc367142f029adeca626f7b8cf765c7a53a73fd138cb data/headers/mainnet_12_7.pk b48d30933a8afefe85f5eafe26d9a5a82a0722ad12a0314768ce744f4351ccf3 data/headers/mainnet_13_7.pk dd2b7a81fa18ec638a288e289eb5e9453c101cecb478e8924d1bc0fad7c6095e data/headers/mainnet_14_7.pk 7b1d5957b070ca1b6e456cca61f129b856cb41ca5738684fc4b45302d8948705 data/headers/mainnet_15_7.pk 88f153a1750cfca2e8fb021e3bcc8441506c8764a58370f1afb6239b125462ae data/headers/mainnet_16_7.pk 1a6be85f70cd0c9651d61c756c9959a0951830329a1e3c78a3372bc130d23ebb data/headers/mainnet_17_7_final.pk a2efd68fbb313d476b964f6f4ad918d3c3717e107fa668f937aa986c9143c346 data/headers/mainnet_17_7_for_evm_0.pk fc80f3468ce0df0247077cd3d4520b4f9ba654e077f8b9de31883e365836e5dc data/headers/mainnet_17_7_for_evm_1.pk f1fc7cb04fef1ce523affcb3e344b544bf40dce2443acc6b6914137ba6f4204b data/headers/mainnet_7.pk 648c02a64380208b2561d405c174559c95fa815cbc33e19892446f24624ae160 data/headers/mainnet_8_7.pk f106d4bfbaa4673b21e0d906bc89dd67791dbc748ead0d4d63ec51b8eb7c4922 data/headers/mainnet_9_7.pk bec832e03debbd5ab484d53c084bf0da73354fb4c731d32a588e234d27bec410 data/headers/mainnet_10_7_for_evm_1.yul f6e0fd84055585bfce8662affe9982158d8523996881a79e5b5119723d5bbf62 data/headers/mainnet_17_7_for_evm_1.yul * feat(block_headers): add shasums.txt file for pkeys and yul Proving keys (which include verifier keys) and on-chain verifier Yul code (which depends on verifier keys) are fixed in production. These SHA sums should not change **unless the SNARKs themselves are being upgraded**. * update(block_headers): new config with 1 column final circuit This only changes `mainnet_10_7_for_evm_0`, `mainnet_10_7_for_evm_1` proving keys and the snark verifier `mainnet_10_7_for_evm_1.yul`. Previously did not notice that the final EVM verifier circuit had 2 advice columns. Changed configs slightly so the circuit has 1 column. This has a difference of around 40_000 gas. * feat(batch_query): new config and yul files Updated configs for batch_query, accompanied by shasums file: 4ad2fc71e23f9a6bc8eb424cbb6d07e3b2f540599827841d4a07fd0a92609c51 data/tests/batch_query/account_3.pk 2dd100517825060bda1f1c51db6b09469b79d91f97a23ff62357105b32cef6ec data/tests/batch_query/account_3_6_0.pk 9c9b2224250ebd807042aaa40dd17527faf8295fe269e7dfa9406d648e1ef9fd data/tests/batch_query/account_3_6_1.pk ba448e91eeb1ce05932da3f98c2c2472aadb675a12749a5c072b7a1d83d9fa34 data/tests/batch_query/account_3_6_2.pk 1bc5f620efaa8057ad249447bcf18703f5137dd0f9893e38349083d9325ceb27 data/tests/batch_query/account_4.pk 67a147d2fbfeecdf6693cc67b7dac1171463e383a867304badc80ed0af9329fd data/tests/batch_query/account_5.pk 588fa07245b64f637059bb89355cb9e3621ba6395db81233ea3e82ea1a8c9256 data/tests/batch_query/final_0.pk 0d348f7589361df4eaac888bcc026680e93d6c74c8e1263b15055580462d27b1 data/tests/batch_query/final_1.pk cc4b26fd45b33f106ee1773f22b86becf79543d88e16bb7723fe7555e86ab03d data/tests/batch_query/final_2.pk e2039ac72d4b50ce68f2cdc55226c2ee263bfe1a2e4cf80da0536ea92510e201 data/tests/batch_query/mainnet_block_6.pk fd8fb7e1b5a444b667bf8070478d1d61d0025635ed9ba319a295b46fe0b6daba data/tests/batch_query/row_6.pk 07b6005328ef68726b2ca27ea702897f4793b8c8bfc7f3c4a3e8990d7cb7d317 data/tests/batch_query/storage_3.pk c4dbd4f5de408b60aca8846c1d55384194b754b214c2bec46bad838d49d41488 data/tests/batch_query/storage_3_6_0.pk b3e8db6bd2d4ef01a43bb836d7177a7c7e904f28581a8ab7d8a0d61ce1f86817 data/tests/batch_query/storage_3_6_1.pk 241b6b057280f109a650fe7f904ca4973c7d77577f828f9d83645e2eb0922357 data/tests/batch_query/storage_3_6_2.pk 587a0c31e98abd8bb2c3cfef33094943e6d4a0b3677b4d638181ae2e150050ac data/tests/batch_query/storage_4.pk 4e30f6cc658a0c239cdaa54b2b631b90c00805361a128721c81dca435241522b data/tests/batch_query/storage_5.pk c1717278590ad66f0b1247c21992d6e254f464984d793e7b938a07f10f08058a data/tests/batch_query/final_2.yul --------- Co-authored-by: Yi Sun <[email protected]> * feat: update readme (axiom-crypto#9) * feat: use retry Http provider switch all functions to use `JsonRpcClient` trait instead of hardcoded `Http` * chore: increment version to 0.2.1 * chore: move src/ into axiom-eth/ * fix: previous merge and RLC builder unwrap error RLC circuit without lookups would panic when unwrapping `lookup_advice` * chore: add community-edition to CI * fix: add missing env variable for RLP test --------- Co-authored-by: Yi Sun <[email protected]>
tideofwords · Jul 1, 2023 · ecbb1c3 · ecbb1c3
1 parent 86934c1
commit ecbb1c3
Show file tree

Hide file tree

Showing 165 changed files with 12,497 additions and 1,633 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,54 @@
+name: Tests
+
+on:
+  push:
+    branches: ["main", "community-edition"]
+  pull_request:
+    branches: ["main", "community-edition"]
+
+env:
+  CARGO_TERM_COLOR: always
+  INFURA_ID: ${{ vars.INFURA_ID }}
+  JSON_RPC_URL: ${{ vars.JSON_RPC_URL }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest-m
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build
+        run: cargo build --verbose
+      - name: Run axiom-eth tests MockProver
+        run: |
+          export INFURA_ID=${{ vars.INFURA_ID }}
+          export JSON_RPC_URL=${{ vars.JSON_RPC_URL }}
+          cd axiom-eth
+          cargo test -- test_mock_account_queries_empty
+          cargo test -- test_mock_account_queries_genesis
+          cargo test -- test_mock_account_queries_simple
+          cargo test -- test_genesis_block
+          cargo test -- test_mock_block_queries_random
+          cargo test -- test_mock_row_consistency_nouns
+          cargo test -- test_default_storage_query
+          cargo test -- test_mock_storage_queries_empty
+          cargo test -- test_mock_storage_queries_mapping
+          cargo test -- test_mock_storage_queries_slot0
+          cargo test -- test_mock_storage_queries_uni_v3
+          cargo test -- test_multi_goerli_header_mock
+          cargo test -- test_one_mainnet_header_before_london_mock
+          cargo test -- test_one_mainnet_header_mock
+          cargo test -- test_one_mainnet_header_withdrawals_mock
+          cargo test -- test_keccak
+          cargo test -- test_mock_mpt_inclusion_fixed
+          cargo test -- test_mpt_empty_root
+          cargo test -- test_mpt_noninclusion_branch_fixed
+          cargo test -- test_mpt_noninclusion_extension_fixed
+          cargo test -- test_mock_rlc
+          cargo test -- test_mock_rlp_array
+          cargo test -- test_mock_rlp_field
+          cargo test -- test_mock_rlp_literal
+          cargo test -- test_mock_rlp_long_field
+          cargo test -- test_mock_rlp_long_long_field
+          cargo test -- test_mock_rlp_short_field
+          cargo test -- test_mock_single_eip1186
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,17 @@
-**/target
+# Generated by Cargo
+# will have compiled files and executables
+/target/
+
+# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
+# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
 Cargo.lock
+
+# These are backup files generated by rustfmt
 **/*.rs.bk
-*~
+=======
+/target
+
+*.png
+
+/halo2_ecc/src/bn254/data/
+/halo2_ecc/src/secp256k1/data/
diff --git a/.gitmodules b/.gitmodules
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,71 +1,39 @@
-[package]
-name = "axiom-eth"
-version = "0.1.1"
-edition = "2021"
-autobins = false
-
-[[bin]]
-name = "header_chain"
-required-features = ["aggregation", "clap"]
-
-[[bin]]
-name = "storage_proof"
-required-features = ["aggregation", "clap", "evm"]
-
-[dependencies]
-itertools = "0.10"
-lazy_static = "1.4.0"
-serde = { version = "1.0", default-features = false, features = ["derive"] }
-serde_json = { version = "1.0", default-features = false }
-rayon = "1.7"
-
-# misc
-log = "0.4"
-env_logger = "0.10"
-ark-std = { version = "0.3.0", features = ["print-trace"], optional = true }
-clap = { version = "4.0.13", features = ["derive"], optional = true }
-clap-num = { version = "1.0.2", optional = true }
-bincode = { version = "1.3.3", optional = true }
-base64 = { version = "0.21", optional = true }
-serde_with = { version = "2.2", optional = true }
-
-# halo2
-ff = "0.12"
-halo2-base = { git = "https://github.com/axiom-crypto/halo2-lib.git", branch = "community-edition", default-features = false }
-zkevm-keccak = { git = "https://github.com/axiom-crypto/halo2-lib.git", branch = "community-edition", default-features = false }
-
-# crypto
-rlp = "0.5.2"
-ethers-core = { version = "=2.0.6" } # used by halo2-mpt already
-rand_core = { version = "0.6", default-features = false, features = ["getrandom"] }
-rand = "0.8"
-rand_chacha = "0.3.1"
-
-# aggregation 
-snark-verifier = { git = "https://github.com/axiom-crypto/snark-verifier.git", branch = "community-edition", default-features = false, features = ["loader_halo2"], optional = true }
-snark-verifier-sdk = { git = "https://github.com/axiom-crypto/snark-verifier.git", branch = "community-edition", default-features = false, features = ["loader_halo2"], optional = true }
-
-# generating circuit inputs from blockchain
-ethers-providers = { version = "2.0.2", optional = true }  
-tokio = { version = "1.26", default-features = false, features = ["rt", "rt-multi-thread"], optional = true }
-
-[dev-dependencies]
-hex = "0.4.3"
-ark-std = { version = "0.3.0", features = ["print-trace"] }
-log = "0.4"
-test-log = "0.2.11"
-test-case = "3.1.0"
-
-[features]
-default = ["halo2-axiom", "jemallocator", "display", "aggregation", "clap", "evm"]
-aggregation = ["dep:snark-verifier", "snark-verifier-sdk", "providers"]
-evm = ["snark-verifier-sdk?/loader_evm", "aggregation"]
-providers = ["dep:ethers-providers", "dep:tokio", "dep:bincode", "dep:base64", "dep:serde_with"]
-display = ["zkevm-keccak/display", "snark-verifier-sdk?/display", "dep:ark-std"]
-clap = ["dep:clap", "dep:clap-num"]
-# 'production' feature turns off circuit auto-configuration and forces trusted setup SRS to be read (and not generated)
-production = []
-# EXACTLY one of halo2-pse / halo2-axiom should always be turned on
-halo2-pse = ["zkevm-keccak/halo2-pse", "snark-verifier-sdk?/halo2-pse"]
-halo2-axiom = ["zkevm-keccak/halo2-axiom", "snark-verifier-sdk?/halo2-axiom"]
-jemallocator = ["halo2-base/jemallocator"]
+[workspace]
+members = [
+    "axiom-eth",
+    "any_circuit_derive"
+]
+
+[profile.dev]
+opt-level = 3
+debug = 1 # change to 0 or 2 for more or less debug info
+overflow-checks = true
+incremental = true
+
+# Local "release" mode, more optimized than dev but faster to compile than release
+[profile.local]
+inherits = "dev"
+opt-level = 3
+# Set this to 1 or 2 to get more useful backtraces
+debug = 1
+debug-assertions = true 
+panic = 'unwind'
+# better recompile times
+incremental = true
+lto = "thin"
+codegen-units = 16
+
+[profile.release]
+opt-level = 3
+debug = false
+debug-assertions = false
+lto = "fat" 
+# `codegen-units = 1` can lead to WORSE performance - always bench to find best profile for your machine!
+# codegen-units = 1
+panic = "abort"
+incremental = false
+
+# For performance profiling
+[profile.flamegraph]
+inherits = "release"
+debug = true
diff --git a/README.md b/README.md
@@ -0,0 +1,3 @@
+# axiom-eth
+
+This repository builds on [halo2-lib](https://github.com/axiom-crypto/halo2-lib/releases/tag/v0.3.0) and [snark-verifier](https://github.com/axiom-crypto/snark-verifier/tree/v0.1.1) to create a library of zero-knowledge proof circuits that [Axiom](https://axiom.xyz) uses to read data from the Ethereum virtual machine (EVM).
diff --git a/any_circuit_derive/Cargo.toml b/any_circuit_derive/Cargo.toml
@@ -0,0 +1,12 @@
+[package]
+name = "any_circuit_derive"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+syn = "2.0.15"
+quote = "1.0.26"
+proc-macro2 = "1.0.56"
diff --git a/any_circuit_derive/src/lib.rs b/any_circuit_derive/src/lib.rs
@@ -0,0 +1,100 @@
+extern crate proc_macro;
+
+use proc_macro::TokenStream;
+use quote::quote;
+use syn::{parse_macro_input, Data, DeriveInput};
+
+#[proc_macro_derive(AnyCircuit)]
+pub fn any_circuit_derive(input: TokenStream) -> TokenStream {
+    let input = parse_macro_input!(input as DeriveInput);
+    let name = &input.ident;
+    let generics = &input.generics;
+    let (impl_generics, ty_generics, where_clause) = generics.split_for_impl();
+
+    let variants = match input.data {
+        Data::Enum(data_enum) => data_enum.variants,
+        _ => panic!("AnyCircuit can only be derived for enums"),
+    };
+
+    let read_or_create_pk_arms = variants.iter().map(|variant| {
+        let ident = &variant.ident;
+        quote! {
+            Self::#ident(pre_circuit) => pre_circuit.read_or_create_pk(params, pk_path, pinning_path, read_only)
+        }
+    });
+
+    let gen_snark_shplonk_arms = variants.iter().map(|variant| {
+        let ident = &variant.ident;
+        quote! {
+            Self::#ident(pre_circuit) => pre_circuit.gen_snark_shplonk(params, pk, pinning_path, path)
+        }
+    });
+
+    let gen_evm_verifier_shplonk_arms = variants.iter().map(|variant| {
+        let ident = &variant.ident;
+        quote! {
+            Self::#ident(pre_circuit) => pre_circuit.gen_evm_verifier_shplonk(params, pk, yul_path)
+        }
+    });
+
+    let gen_calldata_arms = variants.iter().map(|variant| {
+        let ident = &variant.ident;
+        quote! {
+            Self::#ident(pre_circuit) => pre_circuit.gen_calldata(params, pk, pinning_path, path, deployment_code)
+        }
+    });
+
+    let expanded = quote! {
+        impl #impl_generics AnyCircuit for #name #ty_generics #where_clause {
+            fn read_or_create_pk(
+                self,
+                params: &ParamsKZG<Bn256>,
+                pk_path: impl AsRef<Path>,
+                pinning_path: impl AsRef<Path>,
+                read_only: bool,
+            ) -> ProvingKey<G1Affine> {
+                match self {
+                    #(#read_or_create_pk_arms,)*
+                }
+            }
+
+            fn gen_snark_shplonk(
+                self,
+                params: &ParamsKZG<Bn256>,
+                pk: &ProvingKey<G1Affine>,
+                pinning_path: impl AsRef<Path>,
+                path: Option<impl AsRef<Path>>,
+            ) -> Snark {
+                match self {
+                    #(#gen_snark_shplonk_arms,)*
+                }
+            }
+
+            fn gen_evm_verifier_shplonk(
+                self,
+                params: &ParamsKZG<Bn256>,
+                pk: &ProvingKey<G1Affine>,
+                yul_path: impl AsRef<Path>,
+            ) -> Vec<u8> {
+                match self {
+                    #(#gen_evm_verifier_shplonk_arms,)*
+                }
+            }
+
+            fn gen_calldata(
+                self,
+                params: &ParamsKZG<Bn256>,
+                pk: &ProvingKey<G1Affine>,
+                pinning_path: impl AsRef<Path>,
+                path: impl AsRef<Path>,
+                deployment_code: Option<Vec<u8>>,
+            ) -> String {
+                match self {
+                    #(#gen_calldata_arms,)*
+                }
+            }
+        }
+    };
+
+    TokenStream::from(expanded)
+}
diff --git a/axiom-eth/.env.example b/axiom-eth/.env.example
@@ -0,0 +1,2 @@
+# need to export for tests to work
+export JSON_RPC_URL=
diff --git a/axiom-eth/.gitignore b/axiom-eth/.gitignore
@@ -0,0 +1,4 @@
+**/target
+Cargo.lock
+**/*.rs.bk
+*~
diff --git a/axiom-eth/Cargo.toml b/axiom-eth/Cargo.toml
@@ -0,0 +1,69 @@
+[package]
+name = "axiom-eth"
+version = "0.2.1"
+edition = "2021"
+
+[[bin]]
+name = "header_chain"
+required-features = ["aggregation", "clap"]
+
+[dependencies]
+itertools = "0.10"
+lazy_static = "1.4.0"
+serde = { version = "1.0", default-features = false, features = ["derive"] }
+serde_json = { version = "1.0", default-features = false }
+rayon = "1.7"
+
+# misc
+log = "0.4"
+env_logger = "0.10"
+ark-std = { version = "0.3.0", features = ["print-trace"], optional = true }
+clap = { version = "4.0.13", features = ["derive"], optional = true }
+clap-num = { version = "1.0.2", optional = true }
+bincode = { version = "1.3.3", optional = true }
+base64 = { version = "0.21", optional = true }
+serde_with = { version = "2.2", optional = true }
+
+# halo2
+ff = "0.12"
+halo2-base = { git = "https://github.com/axiom-crypto/halo2-lib.git", tag = "v0.3.0", default-features = false }
+zkevm-keccak = { git = "https://github.com/axiom-crypto/halo2-lib.git", tag = "v0.3.0", default-features = false }
+# macro
+any_circuit_derive = { path = "../any_circuit_derive", optional = true }
+
+# crypto
+rlp = "=0.5.2"
+ethers-core = { version = "=2.0.6" }
+rand_core = { version = "0.6", default-features = false, features = ["getrandom"] }
+rand = "0.8"
+rand_chacha = "0.3.1"
+
+# aggregation 
+snark-verifier = { git = "https://github.com/axiom-crypto/snark-verifier.git", tag = "v0.1.1", default-features = false, features = ["loader_halo2"], optional = true }
+snark-verifier-sdk = { git = "https://github.com/axiom-crypto/snark-verifier.git", tag = "v0.1.1", default-features = false, features = ["loader_halo2"], optional = true }
+
+# generating circuit inputs from blockchain
+ethers-providers = { version = "=2.0.6", optional = true }
+tokio = { version = "=1.28", default-features = false, features = ["rt", "rt-multi-thread"], optional = true }
+futures = { version = "=0.3", optional = true }
+
+[dev-dependencies]
+hex = "0.4.3"
+ark-std = { version = "0.3.0", features = ["print-trace"] }
+log = "0.4"
+test-log = "0.2.11"
+test-case = "3.1.0"
+
+[features]
+default = ["halo2-axiom", "jemallocator", "display", "aggregation", "clap", "evm"]
+aggregation = ["dep:snark-verifier", "snark-verifier-sdk", "providers", "dep:any_circuit_derive"]
+evm = ["snark-verifier-sdk?/loader_evm", "aggregation"]
+providers = ["dep:ethers-providers", "dep:tokio", "dep:bincode", "dep:base64", "dep:serde_with", "dep:futures"]
+display = ["zkevm-keccak/display", "snark-verifier-sdk?/display", "dep:ark-std"]
+clap = ["dep:clap", "dep:clap-num"]
+# 'production' feature turns off circuit auto-configuration and forces trusted setup SRS to be read (and not generated)
+production = []
+# EXACTLY one of halo2-pse / halo2-axiom should always be turned on
+halo2-pse = ["zkevm-keccak/halo2-pse", "snark-verifier-sdk?/halo2-pse"]
+halo2-axiom = ["zkevm-keccak/halo2-axiom", "snark-verifier-sdk?/halo2-axiom"]
+jemallocator = ["halo2-base/jemallocator"]