From 2b35ae8df64340c3c7fea8ac4fc594d89227a691 Mon Sep 17 00:00:00 2001 From: Brechtpd Date: Tue, 7 May 2024 02:50:02 +0200 Subject: [PATCH 01/18] Run general tests on all targets --- .github/workflows/ci.yml | 2 +- host/src/raiko.rs | 50 +++++++++++++++++++++++++++------ provers/risc0/driver/src/lib.rs | 13 +-------- scripts/run.sh | 9 ++++-- 4 files changed, 49 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a40b4a0ce..80e6c840d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,7 +78,7 @@ jobs: - name: Build sgx prover run: TARGET="sgx" make build - name: Test sgx prover - run: TARGET="sgx" make test + run: TARGET="sgx" SGX_DIRECT=1 make test build-test-sgx-with-docker: name: Build and test sgx with Docker diff --git a/host/src/raiko.rs b/host/src/raiko.rs index 59a4f38b6..ebead9266 100644 --- a/host/src/raiko.rs +++ b/host/src/raiko.rs @@ -214,12 +214,47 @@ mod tests { use crate::request::{ProofRequest, ProofType}; use crate::rpc_provider::RpcBlockDataProvider; use alloy_primitives::Address; + use clap::ValueEnum; use raiko_lib::{ consts::{get_network_spec, Network}, input::GuestOutput, }; use raiko_primitives::B256; + use serde_json::{json, Value}; use std::collections::HashMap; + use std::env; + + fn get_proof_type_from_env() -> ProofType { + let proof_type = env::var("TARGET").unwrap_or("native".to_string()); + ProofType::from_str(&proof_type, true).unwrap() + } + + fn test_proof_params() -> HashMap { + let mut prover_args = HashMap::new(); + prover_args.insert( + "risc0".to_string(), + json! { + { + "bonsai": false, + "snark": false, + "profile": true, + "execution_po2": 18 + } + }, + ); + prover_args.insert( + "sgx".to_string(), + json! { + { + "instance_id": 121, + "setup": true, + "bootstrap": true, + "prove": true, + } + }, + ); + prover_args + } async fn prove_block(chain_spec: ChainSpec, proof_request: ProofRequest) { let provider = @@ -234,15 +269,11 @@ mod tests { .prove(input, &output) .await .expect("proof generation failed"); - let response: NativeResponse = serde_json::from_value(proof).unwrap(); - match response.output { - GuestOutput::Success(_) => {} - GuestOutput::Failure => unreachable!(), - }; } #[tokio::test(flavor = "multi_thread")] async fn test_prove_block_taiko_a7() { + let proof_type = get_proof_type_from_env(); let network = Network::TaikoA7; let block_number = 39367; let chain_spec = get_network_spec(network); @@ -255,14 +286,15 @@ mod tests { graffiti: B256::ZERO, prover: Address::ZERO, l1_network: Network::Ethereum.to_string(), - proof_type: ProofType::Native, - prover_args: HashMap::new(), + proof_type, + prover_args: test_proof_params(), }; prove_block(chain_spec, proof_request).await; } #[tokio::test(flavor = "multi_thread")] async fn test_prove_block_ethereum() { + let proof_type = get_proof_type_from_env(); let network = Network::Ethereum; let block_number = 19707175; let chain_spec = get_network_spec(network); @@ -275,8 +307,8 @@ mod tests { graffiti: B256::ZERO, prover: Address::ZERO, l1_network: Network::Ethereum.to_string(), - proof_type: ProofType::Native, - prover_args: HashMap::new(), + proof_type, + prover_args: test_proof_params(), }; prove_block(chain_spec, proof_request).await; } diff --git a/provers/risc0/driver/src/lib.rs b/provers/risc0/driver/src/lib.rs index f33f3bc39..e787c8954 100644 --- a/provers/risc0/driver/src/lib.rs +++ b/provers/risc0/driver/src/lib.rs @@ -51,18 +51,7 @@ impl Prover for Risc0Prover { output: &GuestOutput, config: &ProverConfig, ) -> ProverResult { - println!("config: {:?}", config); - - let config_risc0 = json! { - { - "bonsai": false, - "snark": false, - "profile": true, - "execution_po2": 18 - } - }; - - let config = Risc0Param::deserialize(config_risc0).unwrap(); + let config = Risc0Param::deserialize(config).unwrap(); println!("elf code length: {}", RISC0_GUEST_ELF.len()); let encoded_input = to_vec(&input).expect("Could not serialize proving input!"); diff --git a/scripts/run.sh b/scripts/run.sh index 8b5c563bb..715d92a9c 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -21,7 +21,8 @@ if [ -z "$1" ] || [ "$1" == "sgx" ]; then if [ -z "${TEST}" ]; then cargo ${TOOLCHAIN_SGX} ${COMMAND} ${FLAGS} --features sgx else - cargo ${TOOLCHAIN_SGX} test ${FLAGS} -p sgx-prover --features enable + #cargo ${TOOLCHAIN_SGX} test ${FLAGS} -p sgx-prover --features enable + cargo ${TOOLCHAIN_SGX} test ${FLAGS} --features sgx fi fi # RISC0 @@ -30,7 +31,8 @@ if [ -z "$1" ] || [ "$1" == "risc0" ]; then cargo ${TOOLCHAIN_RISC0} run --bin risc0-builder cargo ${TOOLCHAIN_RISC0} ${COMMAND} ${FLAGS} --features risc0 else - RISC0_DEV_MODE=1 cargo ${TOOLCHAIN_RISC0} test ${FLAGS} -p risc0-driver --features enable + #RISC0_DEV_MODE=1 cargo ${TOOLCHAIN_RISC0} test ${FLAGS} -p risc0-driver --features enable + RISC0_DEV_MODE=1 cargo ${TOOLCHAIN_RISC0} test ${FLAGS} --features risc0 fi fi # SP1 @@ -39,6 +41,7 @@ if [ -z "$1" ] || [ "$1" == "sp1" ]; then cargo ${TOOLCHAIN_SP1} run --bin sp1-builder cargo ${TOOLCHAIN_SP1} ${COMMAND} ${FLAGS} --features sp1 else - SP1_PROVER=mock cargo ${TOOLCHAIN_SP1} test ${FLAGS} -p sp1-driver --features enable + #SP1_PROVER=mock cargo ${TOOLCHAIN_SP1} test ${FLAGS} -p sp1-driver --features enable + SP1_PROVER=mock cargo ${TOOLCHAIN_SP1} test ${FLAGS} --features sp1 fi fi \ No newline at end of file From c7d8934b507c4a4f89e07bd1f98b972cbd2de0fc Mon Sep 17 00:00:00 2001 From: Brechtpd Date: Tue, 7 May 2024 03:00:13 +0200 Subject: [PATCH 02/18] fix risc0 --- provers/risc0/driver/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provers/risc0/driver/src/lib.rs b/provers/risc0/driver/src/lib.rs index e787c8954..7cf0cd935 100644 --- a/provers/risc0/driver/src/lib.rs +++ b/provers/risc0/driver/src/lib.rs @@ -51,7 +51,7 @@ impl Prover for Risc0Prover { output: &GuestOutput, config: &ProverConfig, ) -> ProverResult { - let config = Risc0Param::deserialize(config).unwrap(); + let config = Risc0Param::deserialize(config.get("risc0").unwrap()).unwrap(); println!("elf code length: {}", RISC0_GUEST_ELF.len()); let encoded_input = to_vec(&input).expect("Could not serialize proving input!"); From 2ad3ce9d991756243269b1dd18ee5a85d5c245e2 Mon Sep 17 00:00:00 2001 From: Brechtpd Date: Tue, 7 May 2024 03:16:02 +0200 Subject: [PATCH 03/18] fix SGX + temp disable driver tests --- provers/risc0/driver/src/lib.rs | 1 + provers/sgx/prover/src/lib.rs | 8 +++++++- provers/sp1/guest/src/main.rs | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/provers/risc0/driver/src/lib.rs b/provers/risc0/driver/src/lib.rs index 7cf0cd935..cf58b7e07 100644 --- a/provers/risc0/driver/src/lib.rs +++ b/provers/risc0/driver/src/lib.rs @@ -95,6 +95,7 @@ impl Prover for Risc0Prover { } } +#[ignore] #[test] fn test_foo() { use risc0_zkvm::{default_prover, ExecutorEnv}; diff --git a/provers/sgx/prover/src/lib.rs b/provers/sgx/prover/src/lib.rs index c7ea9ef2b..7b3c234ae 100644 --- a/provers/sgx/prover/src/lib.rs +++ b/provers/sgx/prover/src/lib.rs @@ -80,11 +80,17 @@ impl Prover for SgxProver { ); // The working directory - let cur_dir = env::current_exe() + let mut cur_dir = env::current_exe() .expect("Fail to get current directory") .parent() .unwrap() .to_path_buf(); + + // When running in tests we might be in a child folder + if cur_dir.ends_with("deps") { + cur_dir = cur_dir.parent().unwrap().to_path_buf(); + } + println!("Current directory: {cur_dir:?}\n"); // Working paths PRIVATE_KEY diff --git a/provers/sp1/guest/src/main.rs b/provers/sp1/guest/src/main.rs index b0fc9af57..547f308a4 100644 --- a/provers/sp1/guest/src/main.rs +++ b/provers/sp1/guest/src/main.rs @@ -65,6 +65,7 @@ pub fn main() { harness::zk_suits!(test_example); } +#[ignore] #[test] pub fn test_example() { use harness::*; From 8664a5bf4101fb81e6fdcc4256c8c1f9003986df Mon Sep 17 00:00:00 2001 From: Brechtpd Date: Tue, 7 May 2024 05:21:50 +0200 Subject: [PATCH 04/18] typo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80e6c840d..438446dec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,7 @@ jobs: - name: Build sp1 prover run: TARGET="sp1" make build - name: Test sp1 prover - run: TARGET="risc0" make test + run: TARGET="sp1" make test build-test-sgx: name: Build and test sgx From 8006a9bb40d7f294f3e0a21432184a961d5b25c8 Mon Sep 17 00:00:00 2001 From: Brechtpd Date: Tue, 7 May 2024 06:22:11 +0200 Subject: [PATCH 05/18] increase timeout on ci for SP1 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 438446dec..fbe9c0fc7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: build-test-sp1: name: Build and test sp1 runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 120 steps: - uses: actions/checkout@v4 with: From db6fa1e6af4c50aaef044935417531251b97cb4f Mon Sep 17 00:00:00 2001 From: Brechtpd Date: Tue, 7 May 2024 16:02:07 +0200 Subject: [PATCH 06/18] disable slow test on SP1 so ci can handle it --- host/src/raiko.rs | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/host/src/raiko.rs b/host/src/raiko.rs index ebead9266..463920d84 100644 --- a/host/src/raiko.rs +++ b/host/src/raiko.rs @@ -210,15 +210,12 @@ fn check_eq(expected: T, actual: T, me #[cfg(test)] mod tests { - use crate::raiko::{ChainSpec, NativeResponse, Raiko}; + use crate::raiko::{ChainSpec, Raiko}; use crate::request::{ProofRequest, ProofType}; use crate::rpc_provider::RpcBlockDataProvider; use alloy_primitives::Address; use clap::ValueEnum; - use raiko_lib::{ - consts::{get_network_spec, Network}, - input::GuestOutput, - }; + use raiko_lib::consts::{get_network_spec, Network}; use raiko_primitives::B256; use serde_json::{json, Value}; use std::collections::HashMap; @@ -265,7 +262,7 @@ mod tests { .await .expect("input generation failed"); let output = raiko.get_output(&input).expect("output generation failed"); - let proof = raiko + let _proof = raiko .prove(input, &output) .await .expect("proof generation failed"); @@ -295,21 +292,24 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn test_prove_block_ethereum() { let proof_type = get_proof_type_from_env(); - let network = Network::Ethereum; - let block_number = 19707175; - let chain_spec = get_network_spec(network); - let proof_request = ProofRequest { - block_number, - rpc: "https://rpc.ankr.com/eth".to_string(), - l1_rpc: String::new(), - beacon_rpc: String::new(), - network, - graffiti: B256::ZERO, - prover: Address::ZERO, - l1_network: Network::Ethereum.to_string(), - proof_type, - prover_args: test_proof_params(), - }; - prove_block(chain_spec, proof_request).await; + // Skip test on SP1 for now because it's too slow on CI + if proof_type != ProofType::Sp1 { + let network = Network::Ethereum; + let block_number = 19707175; + let chain_spec = get_network_spec(network); + let proof_request = ProofRequest { + block_number, + rpc: "https://rpc.ankr.com/eth".to_string(), + l1_rpc: String::new(), + beacon_rpc: String::new(), + network, + graffiti: B256::ZERO, + prover: Address::ZERO, + l1_network: Network::Ethereum.to_string(), + proof_type, + prover_args: test_proof_params(), + }; + prove_block(chain_spec, proof_request).await; + } } } From 589aaf6f1df55ef874aacd6deb1f01fdabdf0788 Mon Sep 17 00:00:00 2001 From: Brechtpd Date: Tue, 7 May 2024 16:03:05 +0200 Subject: [PATCH 07/18] decrease sp1 timeout again on ci --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fbe9c0fc7..438446dec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: build-test-sp1: name: Build and test sp1 runs-on: ubuntu-latest - timeout-minutes: 120 + timeout-minutes: 60 steps: - uses: actions/checkout@v4 with: From 69375c3c9cce2abea4789fb17a2729133ae78601 Mon Sep 17 00:00:00 2001 From: Brechtpd Date: Tue, 7 May 2024 17:31:15 +0200 Subject: [PATCH 08/18] temp disable sp1 driver test --- provers/sp1/driver/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/provers/sp1/driver/src/lib.rs b/provers/sp1/driver/src/lib.rs index a945aab6b..a388e57d0 100644 --- a/provers/sp1/driver/src/lib.rs +++ b/provers/sp1/driver/src/lib.rs @@ -73,6 +73,7 @@ impl Prover for Sp1Prover { } } +#[ignore] #[test] fn test_example() { let mut client = ProverClient::new(); From f87f7d453ced09a2ed015111f6f79c6d1b093fd0 Mon Sep 17 00:00:00 2001 From: Brechtpd Date: Tue, 7 May 2024 22:02:28 +0200 Subject: [PATCH 09/18] use more recent taiko a7 block --- host/src/raiko.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/host/src/raiko.rs b/host/src/raiko.rs index 463920d84..dfd78b338 100644 --- a/host/src/raiko.rs +++ b/host/src/raiko.rs @@ -272,7 +272,7 @@ mod tests { async fn test_prove_block_taiko_a7() { let proof_type = get_proof_type_from_env(); let network = Network::TaikoA7; - let block_number = 39367; + let block_number = 101368; let chain_spec = get_network_spec(network); let proof_request = ProofRequest { block_number, From 222c5dc6def2acffbf68f0f56fc6dae1dff4df87 Mon Sep 17 00:00:00 2001 From: Brechtpd Date: Wed, 8 May 2024 19:57:15 +0200 Subject: [PATCH 10/18] small improvements + run standard tests with native prover --- .github/workflows/ci.yml | 43 +++++++++++++++++++++++++++-------- README.md | 22 +++++++++--------- host/src/raiko.rs | 6 ++--- provers/sp1/driver/src/lib.rs | 1 - provers/sp1/guest/src/main.rs | 1 - script/build.sh | 38 ++++++++++++++++++++++++++----- 6 files changed, 80 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c74c44e0..52f79a564 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,28 @@ concurrency: cancel-in-progress: true jobs: + build-test: + name: Build and test native + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: actions-rs/toolchain@v1 + with: + toolchain: nightly-2024-04-17 + profile: minimal + - name: Install cargo-binstall + uses: cargo-bins/cargo-binstall@v1.6.4 + - run: export TARGET=native CI=1 MOCK=1 + - name: Install native + run: make install + - name: Build native prover + run: make build + - name: Test native prover + run: make test + build-test-risc0: name: Build and test risc0 runs-on: ubuntu-latest @@ -31,12 +53,13 @@ jobs: - name: Install cargo-binstall uses: cargo-bins/cargo-binstall@v1.6.4 - uses: risc0/risc0/.github/actions/sccache@release-0.19 + - run: export TARGET=risc0 CI=1 MOCK=1 - name: Install risc0 - run: TARGET="risc0" CI=1 make install + run: make install - name: Build risc0 prover - run: TARGET="risc0" make build + run: make build - name: Test risc0 prover - run: MOCK=1 TARGET="risc0" make test + run: make test build-test-sp1: name: Build and test sp1 @@ -52,12 +75,13 @@ jobs: profile: minimal - name: Install cargo-binstall uses: cargo-bins/cargo-binstall@v1.6.4 + - run: export TARGET=sp1 CI=1 MOCK=1 - name: Install sp1 - run: TARGET="sp1" CI=1 make install + run: make install - name: Build sp1 prover - run: TARGET="sp1" make build + run: make build - name: Test sp1 prover - run: MOCK=1 TARGET="sp1" make test + run: make test build-test-sgx: name: Build and test sgx @@ -73,12 +97,13 @@ jobs: profile: minimal - name: Install cargo-binstall uses: cargo-bins/cargo-binstall@v1.6.4 + - run: export TARGET=sgx CI=1 MOCK=1 SGX_DIRECT=1 - name: Install sgx - run: TARGET="sgx" CI=1 make install + run: make install - name: Build sgx prover - run: TARGET="sgx" make build + run: make build - name: Test sgx prover - run: TARGET="sgx" SGX_DIRECT=1 make test + run: make test build-test-sgx-with-docker: name: Build and test sgx with Docker diff --git a/README.md b/README.md index 702d157c6..96df70823 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,13 @@ $ make install You can also download all required dependencies for each prover separately, for example to install SP1: ```console -$ TARGET="sp1" make install +$ TARGET=sp1 make install ``` ### Building After installing dependencies of selected prover, the following command internally calls cargo to build the prover's guest target with the `--release` profile by default, for example: ```console -$ TARGET="sp1" make build +$ TARGET=sp1 make build ``` If you set `DEBUG=1` then the target will be compiled without optimization (not recomended for ZkVM elfs). @@ -29,7 +29,7 @@ If you set `DEBUG=1` then the target will be compiled without optimization (not Note that you have to `make build` first before running ZkVM provers, otherwise the guest elf may not be up to date and can result in poof failures. ```console -$ TARGET="sp1" make run +$ TARGET=sp1 make run ``` Just for development with the native prover which runs through the block execution without producing any ZK/SGX proof: ``` @@ -49,12 +49,12 @@ Look into `prove-block.sh` for the available options or run the script without i ### Risc zero To install, build, and run in one step: ```console -$ export TARGET="risc0" +$ export TARGET=risc0 $ make install && make build && make run ``` To build and run test on Risc0 Zkvm: ```console -$ TARGET="risc0" make test +$ TARGET=risc0 make test ``` #### Bonsai If you are using Bonsai service, edit `run-bonsai.sh` to setup your API key, endpoint and on-chain verifier address. @@ -74,29 +74,29 @@ $ cargo run -F metal --release --features risc0 Note that CUDA needs to be installed when using `cuda`: https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html -### SP1 +### SP1 To install, build, and run in one step: ```console -$ export TARGET="sp1" +$ export TARGET=sp1 $ make install && make build && make run ``` To build and run test on Sp1 Zkvm: ```console -$ TARGET="sp1" make test +$ TARGET=sp1 make test ``` ### SGX: To install, build, and run in one step: ```console -$ export TARGET="sgx" +$ export TARGET=sgx $ make install && make build && make run ``` To build and run test related SGX provers: ```console -$ TARGET="sgx" make test +$ TARGET=sgx make test ``` If your CPU doesn't support SGX, you can still run the SGX code through gramine like it would on an SGX machine: ```console -$ SGX_DIRECT=1 TARGET="sgx" make run +$ SGX_DIRECT=1 TARGET=sgx make run ``` \ No newline at end of file diff --git a/host/src/raiko.rs b/host/src/raiko.rs index dfd78b338..27338156d 100644 --- a/host/src/raiko.rs +++ b/host/src/raiko.rs @@ -272,12 +272,12 @@ mod tests { async fn test_prove_block_taiko_a7() { let proof_type = get_proof_type_from_env(); let network = Network::TaikoA7; - let block_number = 101368; + let block_number = 105987; let chain_spec = get_network_spec(network); let proof_request = ProofRequest { block_number, - rpc: "https://rpc.hekla.taiko.xyz/".to_string(), - l1_rpc: "https://l1rpc.hekla.taiko.xyz/".to_string(), + rpc: "https://rpc.hekla.taiko.xyz".to_string(), + l1_rpc: "https://ethereum-holesky-rpc.publicnode.com".to_string(), beacon_rpc: "https://l1beacon.hekla.taiko.xyz".to_string(), network, graffiti: B256::ZERO, diff --git a/provers/sp1/driver/src/lib.rs b/provers/sp1/driver/src/lib.rs index 561f43be3..59f9fe74b 100644 --- a/provers/sp1/driver/src/lib.rs +++ b/provers/sp1/driver/src/lib.rs @@ -73,7 +73,6 @@ impl Prover for Sp1Prover { } } -#[ignore] #[test] fn test_example() { // TODO(Cecilia): imple GuestInput::mock() for unit test diff --git a/provers/sp1/guest/src/main.rs b/provers/sp1/guest/src/main.rs index 547f308a4..b0fc9af57 100644 --- a/provers/sp1/guest/src/main.rs +++ b/provers/sp1/guest/src/main.rs @@ -65,7 +65,6 @@ pub fn main() { harness::zk_suits!(test_example); } -#[ignore] #[test] pub fn test_example() { use harness::*; diff --git a/script/build.sh b/script/build.sh index 0c3333580..597e844c1 100755 --- a/script/build.sh +++ b/script/build.sh @@ -23,11 +23,11 @@ check_toolchain() { echo "Installing Rust toolchain: $TOOLCHAIN" rustup install "$TOOLCHAIN" fi -} +} if [ -z "${DEBUG}" ]; then FLAGS=--release -else +else echo "Warning: in debug mode" fi @@ -37,23 +37,49 @@ else COMMAND=run fi +# NATIVE +if [ -z "$1" ] || [ "$1" == "native" ]; then + if [ -z "${RUN}" ]; then + if [ -z "${TEST}" ]; then + echo "Building native prover" + cargo build ${FLAGS} + else + echo "Building native tests" + cargo test --no-run + fi + else + if [ -z "${TEST}" ]; then + echo "Running native prover" + cargo run ${FLAGS} + else + echo "Running native tests" + cargo test ${FLAGS} + fi + fi +fi + # SGX if [ -z "$1" ] || [ "$1" == "sgx" ]; then check_toolchain $TOOLCHAIN_SGX if [ -z "${RUN}" ]; then if [ -z "${TEST}" ]; then + echo "Building SGX prover" cargo ${TOOLCHAIN_SGX} build ${FLAGS} --features sgx else - cargo ${TOOLCHAIN_SGX} test ${FLAGS} -p sgx-prover --features enable --no-run + echo "Building SGX tests" + cargo ${TOOLCHAIN_SGX} test ${FLAGS} -p host -p sgx-prover --features enable --no-run fi else if [ -z "${TEST}" ]; then + echo "Running SGX prover" cargo ${TOOLCHAIN_SGX} run ${FLAGS} --features sgx else - cargo ${TOOLCHAIN_SGX} test ${FLAGS} -p sgx-prover --features enable + echo "Running SGX tests" + cargo ${TOOLCHAIN_SGX} test ${FLAGS} -p host -p sgx-prover --features enable fi fi fi + # RISC0 if [ -z "$1" ] || [ "$1" == "risc0" ]; then check_toolchain $TOOLCHAIN_RISC0 @@ -77,7 +103,7 @@ if [ -z "$1" ] || [ "$1" == "risc0" ]; then cargo ${TOOLCHAIN_RISC0} run ${FLAGS} --features risc0 else echo "Running Sp1 tests" - RISC0_DEV_MODE=1 cargo ${TOOLCHAIN_RISC0} test ${FLAGS} -p risc0-driver --features enable + cargo ${TOOLCHAIN_RISC0} test ${FLAGS} -p host -p risc0-driver --features enable fi fi fi @@ -104,7 +130,7 @@ if [ -z "$1" ] || [ "$1" == "sp1" ]; then cargo ${TOOLCHAIN_SP1} run ${FLAGS} --features sp1 else echo "Running Sp1 tests" - cargo ${TOOLCHAIN_SP1} test ${FLAGS} -p sp1-driver --features enable + cargo ${TOOLCHAIN_SP1} test ${FLAGS} -p host -p sp1-driver --features enable fi fi fi \ No newline at end of file From 354b05ad9902fea34279f45a0a1891e161146e4d Mon Sep 17 00:00:00 2001 From: Brechtpd Date: Wed, 8 May 2024 19:59:34 +0200 Subject: [PATCH 11/18] update prove_block with working holesky endpoint --- script/prove-block.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/prove-block.sh b/script/prove-block.sh index bdc780c05..5e72c7996 100755 --- a/script/prove-block.sh +++ b/script/prove-block.sh @@ -30,11 +30,11 @@ elif [ "$chain" == "holesky" ]; then rpc="https://ethereum-holesky-rpc.publicnode.com" elif [ "$chain" == "taiko_a6" ]; then rpc="https://rpc.katla.taiko.xyz" - l1Rpc="https://l1rpc.katla.taiko.xyz" + l1Rpc="https://ethereum-holesky-rpc.publicnode.com" beaconRpc="https://l1beacon.hekla.taiko.xyz" elif [ "$chain" == "taiko_a7" ]; then - rpc="https://rpc.hekla.taiko.xyz/" - l1Rpc="https://l1rpc.hekla.taiko.xyz/" + rpc="https://rpc.hekla.taiko.xyz" + l1Rpc="https://ethereum-holesky-rpc.publicnode.com" beaconRpc="https://l1beacon.hekla.taiko.xyz" else echo "Invalid chain name. Please use 'ethereum', 'taiko_a6' or 'taiko_a7'." From fd64b774f608c60218e0070a6c90a3b75132f76e Mon Sep 17 00:00:00 2001 From: Brechtpd Date: Wed, 8 May 2024 21:14:17 +0200 Subject: [PATCH 12/18] increase timeout --- .github/workflows/ci.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52f79a564..d320791d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,8 +8,6 @@ on: env: CARGO_TERM_COLOR: always GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - RISC0_VERSION: 0.20.1 - RISC0_TOOLCHAIN_VERSION: test-release-2 concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -19,7 +17,7 @@ jobs: build-test: name: Build and test native runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 120 steps: - uses: actions/checkout@v4 with: @@ -41,7 +39,7 @@ jobs: build-test-risc0: name: Build and test risc0 runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 120 steps: - uses: actions/checkout@v4 with: @@ -64,7 +62,7 @@ jobs: build-test-sp1: name: Build and test sp1 runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 120 steps: - uses: actions/checkout@v4 with: @@ -86,7 +84,7 @@ jobs: build-test-sgx: name: Build and test sgx runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 120 steps: - uses: actions/checkout@v4 with: From 230568cfd07bc3964d5c4d4301c2fc08195b9970 Mon Sep 17 00:00:00 2001 From: Brechtpd Date: Wed, 8 May 2024 21:59:28 +0200 Subject: [PATCH 13/18] fix ci env vars --- .github/workflows/ci.yml | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d320791d8..d1ad17eb6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,10 @@ jobs: name: Build and test native runs-on: ubuntu-latest timeout-minutes: 120 + env: + TARGET: native + CI: 1 + MOCK: 1 steps: - uses: actions/checkout@v4 with: @@ -28,7 +32,6 @@ jobs: profile: minimal - name: Install cargo-binstall uses: cargo-bins/cargo-binstall@v1.6.4 - - run: export TARGET=native CI=1 MOCK=1 - name: Install native run: make install - name: Build native prover @@ -40,6 +43,10 @@ jobs: name: Build and test risc0 runs-on: ubuntu-latest timeout-minutes: 120 + env: + TARGET: risc0 + CI: 1 + MOCK: 1 steps: - uses: actions/checkout@v4 with: @@ -51,7 +58,6 @@ jobs: - name: Install cargo-binstall uses: cargo-bins/cargo-binstall@v1.6.4 - uses: risc0/risc0/.github/actions/sccache@release-0.19 - - run: export TARGET=risc0 CI=1 MOCK=1 - name: Install risc0 run: make install - name: Build risc0 prover @@ -63,6 +69,10 @@ jobs: name: Build and test sp1 runs-on: ubuntu-latest timeout-minutes: 120 + env: + TARGET: sp1 + CI: 1 + MOCK: 1 steps: - uses: actions/checkout@v4 with: @@ -73,7 +83,6 @@ jobs: profile: minimal - name: Install cargo-binstall uses: cargo-bins/cargo-binstall@v1.6.4 - - run: export TARGET=sp1 CI=1 MOCK=1 - name: Install sp1 run: make install - name: Build sp1 prover @@ -85,6 +94,11 @@ jobs: name: Build and test sgx runs-on: ubuntu-latest timeout-minutes: 120 + env: + TARGET: sgx + CI: 1 + MOCK: 1 + SGX_DIRECT: 1 steps: - uses: actions/checkout@v4 with: @@ -95,7 +109,6 @@ jobs: profile: minimal - name: Install cargo-binstall uses: cargo-bins/cargo-binstall@v1.6.4 - - run: export TARGET=sgx CI=1 MOCK=1 SGX_DIRECT=1 - name: Install sgx run: make install - name: Build sgx prover From dd3971cc875e524062f36cc1a85b6b2dc3e65a7e Mon Sep 17 00:00:00 2001 From: Brechtpd Date: Wed, 8 May 2024 22:18:12 +0200 Subject: [PATCH 14/18] more fixes to features --- .github/workflows/ci.yml | 1 - README.md | 2 +- script/build.sh | 18 +++++++++++------- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1ad17eb6..f68eb6e35 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,7 +98,6 @@ jobs: TARGET: sgx CI: 1 MOCK: 1 - SGX_DIRECT: 1 steps: - uses: actions/checkout@v4 with: diff --git a/README.md b/README.md index 96df70823..b462255b8 100644 --- a/README.md +++ b/README.md @@ -98,5 +98,5 @@ $ TARGET=sgx make test If your CPU doesn't support SGX, you can still run the SGX code through gramine like it would on an SGX machine: ```console -$ SGX_DIRECT=1 TARGET=sgx make run +$ MOCK=1 TARGET=sgx make run ``` \ No newline at end of file diff --git a/script/build.sh b/script/build.sh index 597e844c1..836f3d7aa 100755 --- a/script/build.sh +++ b/script/build.sh @@ -9,7 +9,7 @@ check_toolchain() { local TOOLCHAIN=$1 # Remove the plus sign from the toolchain name - TOOLCHAIN=${TOOLCHAIN#+} + TOOLCHAIN=${TOOLCHAIN#+} # Function to check if the toolchain is installed exist() { @@ -45,7 +45,7 @@ if [ -z "$1" ] || [ "$1" == "native" ]; then cargo build ${FLAGS} else echo "Building native tests" - cargo test --no-run + cargo test ${FLAGS} --no-run fi else if [ -z "${TEST}" ]; then @@ -61,13 +61,17 @@ fi # SGX if [ -z "$1" ] || [ "$1" == "sgx" ]; then check_toolchain $TOOLCHAIN_SGX + if [ "$MOCK" = "1" ]; then + export SGX_DIRECT=1 + echo "SGX_DIRECT is set to $SGX_DIRECT" + fi if [ -z "${RUN}" ]; then if [ -z "${TEST}" ]; then echo "Building SGX prover" cargo ${TOOLCHAIN_SGX} build ${FLAGS} --features sgx else echo "Building SGX tests" - cargo ${TOOLCHAIN_SGX} test ${FLAGS} -p host -p sgx-prover --features enable --no-run + cargo ${TOOLCHAIN_SGX} test ${FLAGS} -p raiko-host -p sgx-prover --features "sgx enable" --no-run fi else if [ -z "${TEST}" ]; then @@ -75,7 +79,7 @@ if [ -z "$1" ] || [ "$1" == "sgx" ]; then cargo ${TOOLCHAIN_SGX} run ${FLAGS} --features sgx else echo "Running SGX tests" - cargo ${TOOLCHAIN_SGX} test ${FLAGS} -p host -p sgx-prover --features enable + cargo ${TOOLCHAIN_SGX} test ${FLAGS} -p raiko-host -p sgx-prover --features "sgx enable" fi fi fi @@ -103,7 +107,7 @@ if [ -z "$1" ] || [ "$1" == "risc0" ]; then cargo ${TOOLCHAIN_RISC0} run ${FLAGS} --features risc0 else echo "Running Sp1 tests" - cargo ${TOOLCHAIN_RISC0} test ${FLAGS} -p host -p risc0-driver --features enable + cargo ${TOOLCHAIN_RISC0} test ${FLAGS} -p raiko-host -p risc0-driver --features "risc0 enable" fi fi fi @@ -111,7 +115,7 @@ fi # SP1 if [ -z "$1" ] || [ "$1" == "sp1" ]; then check_toolchain $TOOLCHAIN_SP1 - if [ "$MOCK" = "1" ]; then + if [ "$MOCK" = "1" ]; then export SP1_PROVER=mock echo "SP1_PROVER is set to $SP1_PROVER" fi @@ -130,7 +134,7 @@ if [ -z "$1" ] || [ "$1" == "sp1" ]; then cargo ${TOOLCHAIN_SP1} run ${FLAGS} --features sp1 else echo "Running Sp1 tests" - cargo ${TOOLCHAIN_SP1} test ${FLAGS} -p host -p sp1-driver --features enable + cargo ${TOOLCHAIN_SP1} test ${FLAGS} -p raiko-host -p sp1-driver --features "sp1 enable" fi fi fi \ No newline at end of file From e333016bd25b95c8634d8edeee655a28132d8df4 Mon Sep 17 00:00:00 2001 From: Brechtpd Date: Wed, 8 May 2024 22:19:41 +0200 Subject: [PATCH 15/18] Fix build error propegation --- script/build.sh | 2 +- script/install.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/script/build.sh b/script/build.sh index 836f3d7aa..64854a998 100755 --- a/script/build.sh +++ b/script/build.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash +set -e TOOLCHAIN_RISC0=+nightly-2024-02-06 TOOLCHAIN_SP1=+nightly-2024-02-06 TOOLCHAIN_SGX=+nightly-2024-02-06 - check_toolchain() { local TOOLCHAIN=$1 diff --git a/script/install.sh b/script/install.sh index 836b34377..a60b55b56 100755 --- a/script/install.sh +++ b/script/install.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash + +# Any error will result in failure set -e # toolchain necessary to compile c-kzg in SP1/risc0 From ae5c2edcb065201334eaaf8ba9949bd983a38582 Mon Sep 17 00:00:00 2001 From: Brechtpd Date: Wed, 8 May 2024 23:36:49 +0200 Subject: [PATCH 16/18] skip kzg check on SP1 because out of memory problems --- host/src/preflight.rs | 1 + host/src/raiko.rs | 12 ++++++++++-- lib/src/input.rs | 1 + lib/src/protocol_instance.rs | 26 ++++++++++++++++---------- provers/sgx/guest/src/one_shot.rs | 1 + script/build.sh | 2 ++ 6 files changed, 31 insertions(+), 12 deletions(-) diff --git a/host/src/preflight.rs b/host/src/preflight.rs index 14c5095bf..3f76535f9 100644 --- a/host/src/preflight.rs +++ b/host/src/preflight.rs @@ -144,6 +144,7 @@ pub async fn preflight( tx_blob_hash, block_proposed: proposal_event, prover_data, + verify_blob: true, } } else { // For Ethereum blocks we just convert the block transactions in a tx_list diff --git a/host/src/raiko.rs b/host/src/raiko.rs index 27338156d..80ee150c0 100644 --- a/host/src/raiko.rs +++ b/host/src/raiko.rs @@ -226,6 +226,11 @@ mod tests { ProofType::from_str(&proof_type, true).unwrap() } + fn is_ci() -> bool { + let ci = env::var("CI").unwrap_or("0".to_string()); + ci == "1" + } + fn test_proof_params() -> HashMap { let mut prover_args = HashMap::new(); prover_args.insert( @@ -256,11 +261,14 @@ mod tests { async fn prove_block(chain_spec: ChainSpec, proof_request: ProofRequest) { let provider = RpcBlockDataProvider::new(&proof_request.rpc.clone(), proof_request.block_number - 1); - let raiko = Raiko::new(chain_spec, proof_request); - let input = raiko + let raiko = Raiko::new(chain_spec, proof_request.clone()); + let mut input = raiko .generate_input(provider) .await .expect("input generation failed"); + if is_ci() && proof_request.proof_type == ProofType::Sp1 { + input.taiko.verify_blob = false; + } let output = raiko.get_output(&input).expect("output generation failed"); let _proof = raiko .prove(input, &output) diff --git a/lib/src/input.rs b/lib/src/input.rs index 99f548863..9584101db 100644 --- a/lib/src/input.rs +++ b/lib/src/input.rs @@ -94,6 +94,7 @@ pub struct TaikoGuestInput { pub block_proposed: BlockProposed, pub prover_data: TaikoProverData, pub tx_blob_hash: Option, + pub verify_blob: bool, } #[derive(Clone, Default, Debug, Serialize, Deserialize)] diff --git a/lib/src/protocol_instance.rs b/lib/src/protocol_instance.rs index a248979f2..126d4e9c5 100644 --- a/lib/src/protocol_instance.rs +++ b/lib/src/protocol_instance.rs @@ -94,16 +94,22 @@ pub fn assemble_protocol_instance( ) -> Result { let blob_used = input.taiko.block_proposed.meta.blobUsed; let tx_list_hash = if blob_used { - let mut data = Vec::from(KZG_TRUST_SETUP_DATA); - let kzg_settings = KzgSettings::from_u8_slice(&mut data); - let kzg_commit = KzgCommitment::blob_to_kzg_commitment( - &Blob::from_bytes(&input.taiko.tx_data.as_slice()).unwrap(), - &kzg_settings, - ) - .unwrap(); - let versioned_hash = kzg_to_versioned_hash(kzg_commit); - assert_eq!(versioned_hash, input.taiko.tx_blob_hash.unwrap()); - versioned_hash + if input.taiko.verify_blob { + println!("kzg check enabled!"); + let mut data = Vec::from(KZG_TRUST_SETUP_DATA); + let kzg_settings = KzgSettings::from_u8_slice(&mut data); + let kzg_commit = KzgCommitment::blob_to_kzg_commitment( + &Blob::from_bytes(&input.taiko.tx_data.as_slice()).unwrap(), + &kzg_settings, + ) + .unwrap(); + let versioned_hash = kzg_to_versioned_hash(kzg_commit); + assert_eq!(versioned_hash, input.taiko.tx_blob_hash.unwrap()); + versioned_hash + } else { + println!("kzg check disabled!"); + input.taiko.tx_blob_hash.unwrap() + } } else { TxHash::from(keccak(input.taiko.tx_data.as_slice())) }; diff --git a/provers/sgx/guest/src/one_shot.rs b/provers/sgx/guest/src/one_shot.rs index 9fa8fd3ce..cf43f2c44 100644 --- a/provers/sgx/guest/src/one_shot.rs +++ b/provers/sgx/guest/src/one_shot.rs @@ -122,6 +122,7 @@ pub async fn one_shot(global_opts: GlobalOpts, args: OneShotArgs) -> Result<()> let new_instance = public_key_to_address(&new_pubkey); let input = bincode::deserialize_from(std::io::stdin()).expect("unable to deserialize input"); + assert!(input.taiko.verify_blob); // Process the block let (header, _mpt_node) = diff --git a/script/build.sh b/script/build.sh index 64854a998..06eb06fca 100755 --- a/script/build.sh +++ b/script/build.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash + +# Any error will result in failure set -e TOOLCHAIN_RISC0=+nightly-2024-02-06 From 29987f0249d24560e4838ceb107fdd215bfd4479 Mon Sep 17 00:00:00 2001 From: Brechtpd Date: Wed, 8 May 2024 23:54:02 +0200 Subject: [PATCH 17/18] fix sgx compile error --- provers/sgx/guest/src/one_shot.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/provers/sgx/guest/src/one_shot.rs b/provers/sgx/guest/src/one_shot.rs index cf43f2c44..794009cf2 100644 --- a/provers/sgx/guest/src/one_shot.rs +++ b/provers/sgx/guest/src/one_shot.rs @@ -9,6 +9,7 @@ use anyhow::{anyhow, bail, Context, Error, Result}; use base64_serde::base64_serde_type; use raiko_lib::{ builder::{BlockBuilderStrategy, TaikoStrategy}, + input::GuestInput, protocol_instance::{assemble_protocol_instance, EvidenceType}, }; use raiko_primitives::Address; @@ -121,7 +122,8 @@ pub async fn one_shot(global_opts: GlobalOpts, args: OneShotArgs) -> Result<()> let new_pubkey = public_key(&prev_privkey); let new_instance = public_key_to_address(&new_pubkey); - let input = bincode::deserialize_from(std::io::stdin()).expect("unable to deserialize input"); + let input: GuestInput = + bincode::deserialize_from(std::io::stdin()).expect("unable to deserialize input"); assert!(input.taiko.verify_blob); // Process the block From b850a4ed3677fb2abe9c0b0e4b17a0c6bd9b9c04 Mon Sep 17 00:00:00 2001 From: Brechtpd Date: Thu, 9 May 2024 00:18:53 +0200 Subject: [PATCH 18/18] fix default skip value --- host/src/preflight.rs | 2 +- host/src/raiko.rs | 2 +- lib/src/input.rs | 2 +- lib/src/protocol_instance.rs | 2 +- provers/sgx/guest/src/one_shot.rs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/host/src/preflight.rs b/host/src/preflight.rs index 3f76535f9..c48f08bb8 100644 --- a/host/src/preflight.rs +++ b/host/src/preflight.rs @@ -144,7 +144,7 @@ pub async fn preflight( tx_blob_hash, block_proposed: proposal_event, prover_data, - verify_blob: true, + skip_verify_blob: false, } } else { // For Ethereum blocks we just convert the block transactions in a tx_list diff --git a/host/src/raiko.rs b/host/src/raiko.rs index 80ee150c0..80e069684 100644 --- a/host/src/raiko.rs +++ b/host/src/raiko.rs @@ -267,7 +267,7 @@ mod tests { .await .expect("input generation failed"); if is_ci() && proof_request.proof_type == ProofType::Sp1 { - input.taiko.verify_blob = false; + input.taiko.skip_verify_blob = true; } let output = raiko.get_output(&input).expect("output generation failed"); let _proof = raiko diff --git a/lib/src/input.rs b/lib/src/input.rs index 9584101db..7c67706e6 100644 --- a/lib/src/input.rs +++ b/lib/src/input.rs @@ -94,7 +94,7 @@ pub struct TaikoGuestInput { pub block_proposed: BlockProposed, pub prover_data: TaikoProverData, pub tx_blob_hash: Option, - pub verify_blob: bool, + pub skip_verify_blob: bool, } #[derive(Clone, Default, Debug, Serialize, Deserialize)] diff --git a/lib/src/protocol_instance.rs b/lib/src/protocol_instance.rs index 126d4e9c5..a63a2b201 100644 --- a/lib/src/protocol_instance.rs +++ b/lib/src/protocol_instance.rs @@ -94,7 +94,7 @@ pub fn assemble_protocol_instance( ) -> Result { let blob_used = input.taiko.block_proposed.meta.blobUsed; let tx_list_hash = if blob_used { - if input.taiko.verify_blob { + if !input.taiko.skip_verify_blob { println!("kzg check enabled!"); let mut data = Vec::from(KZG_TRUST_SETUP_DATA); let kzg_settings = KzgSettings::from_u8_slice(&mut data); diff --git a/provers/sgx/guest/src/one_shot.rs b/provers/sgx/guest/src/one_shot.rs index 794009cf2..1edf69ebc 100644 --- a/provers/sgx/guest/src/one_shot.rs +++ b/provers/sgx/guest/src/one_shot.rs @@ -124,7 +124,7 @@ pub async fn one_shot(global_opts: GlobalOpts, args: OneShotArgs) -> Result<()> let input: GuestInput = bincode::deserialize_from(std::io::stdin()).expect("unable to deserialize input"); - assert!(input.taiko.verify_blob); + assert!(!input.taiko.skip_verify_blob); // Process the block let (header, _mpt_node) =