From 6253f63122067cde99ca7f40d147aeabbfa55cc8 Mon Sep 17 00:00:00 2001 From: Johnny <132952411+jd-sxt@users.noreply.github.com> Date: Wed, 23 Oct 2024 21:03:39 -0500 Subject: [PATCH 01/10] feat: adding stub for avocado prices example --- crates/proof-of-sql/Cargo.toml | 3 +++ crates/proof-of-sql/examples/avocado-prices/main.rs | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 crates/proof-of-sql/examples/avocado-prices/main.rs diff --git a/crates/proof-of-sql/Cargo.toml b/crates/proof-of-sql/Cargo.toml index 853e2e31f..f0973b312 100644 --- a/crates/proof-of-sql/Cargo.toml +++ b/crates/proof-of-sql/Cargo.toml @@ -119,6 +119,9 @@ required-features = [ "arrow" ] name = "plastics" required-features = [ "arrow" ] +[[example]] +name = "avocado-prices" + [[bench]] name = "posql_benches" harness = false diff --git a/crates/proof-of-sql/examples/avocado-prices/main.rs b/crates/proof-of-sql/examples/avocado-prices/main.rs new file mode 100644 index 000000000..44c925049 --- /dev/null +++ b/crates/proof-of-sql/examples/avocado-prices/main.rs @@ -0,0 +1,3 @@ +//! Example to use Proof of SQL with datasets +//! To run, use `cargo run --example avocado-prices`. +fn main() {} From aef29091af37fe6f6988d67e9560934936f089f5 Mon Sep 17 00:00:00 2001 From: Johnny <132952411+jd-sxt@users.noreply.github.com> Date: Wed, 23 Oct 2024 21:04:05 -0500 Subject: [PATCH 02/10] feat: add dataset for example --- .../avocado-prices/avocado-prices.csv | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 crates/proof-of-sql/examples/avocado-prices/avocado-prices.csv diff --git a/crates/proof-of-sql/examples/avocado-prices/avocado-prices.csv b/crates/proof-of-sql/examples/avocado-prices/avocado-prices.csv new file mode 100644 index 000000000..a5b94b592 --- /dev/null +++ b/crates/proof-of-sql/examples/avocado-prices/avocado-prices.csv @@ -0,0 +1,36 @@ +Year,Price +1990,0.96 +1991,1.0 +1992,2.69 +1993,1.49 +1994,1.27 +1995,1.53 +1996,2.32 +1997,1.27 +1998,2.49 +1999,2.4 +2000,2.41 +2001,0.9 +2002,0.91 +2003,1.69 +2004,1.67 +2005,0.56 +2006,2.3 +2007,1.74 +2008,1.24 +2009,0.92 +2010,2.01 +2011,1.67 +2012,1.25 +2013,1.47 +2014,2.85 +2015,1.54 +2016,1.06 +2017,2.23 +2018,0.85 +2019,1.45 +2020,1.47 +2021,0.68 +2022,1.42 +2023,2.81 +2024,1.64 From 5660de25af88b0a33493515a18988bf3b143655c Mon Sep 17 00:00:00 2001 From: Johnny <132952411+jd-sxt@users.noreply.github.com> Date: Wed, 23 Oct 2024 21:04:33 -0500 Subject: [PATCH 03/10] fix: multiply price by 100 to eliminate float --- .../avocado-prices/avocado-prices.csv | 71 ++++++++++--------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/crates/proof-of-sql/examples/avocado-prices/avocado-prices.csv b/crates/proof-of-sql/examples/avocado-prices/avocado-prices.csv index a5b94b592..7750f7a46 100644 --- a/crates/proof-of-sql/examples/avocado-prices/avocado-prices.csv +++ b/crates/proof-of-sql/examples/avocado-prices/avocado-prices.csv @@ -1,36 +1,37 @@ Year,Price -1990,0.96 -1991,1.0 -1992,2.69 -1993,1.49 -1994,1.27 -1995,1.53 -1996,2.32 -1997,1.27 -1998,2.49 -1999,2.4 -2000,2.41 -2001,0.9 -2002,0.91 -2003,1.69 -2004,1.67 -2005,0.56 -2006,2.3 -2007,1.74 -2008,1.24 -2009,0.92 -2010,2.01 -2011,1.67 -2012,1.25 -2013,1.47 -2014,2.85 -2015,1.54 -2016,1.06 -2017,2.23 -2018,0.85 -2019,1.45 -2020,1.47 -2021,0.68 -2022,1.42 -2023,2.81 -2024,1.64 +1990,96 +1991,100 +1992,269 +1993,149 +1994,127 +1995,153 +1996,232 +1997,127 +1998,249 +1999,240 +2000,241 +2001,90 +2002,91 +2003,169 +2004,167 +2005,56 +2006,230 +2007,174 +2008,124 +2009,92 +2010,201 +2011,167 +2012,125 +2013,147 +2014,285 +2015,154 +2016,106 +2017,223 +2018,85 +2019,145 +2020,147 +2021,68 +2022,142 +2023,281 +2024,164 + From c9956ee19dfcee5910f7653c6412e9bee8636018 Mon Sep 17 00:00:00 2001 From: Johnny <132952411+jd-sxt@users.noreply.github.com> Date: Wed, 23 Oct 2024 21:04:26 -0500 Subject: [PATCH 04/10] feat: load csv file in example --- crates/proof-of-sql/Cargo.toml | 1 + .../examples/avocado-prices/main.rs | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/crates/proof-of-sql/Cargo.toml b/crates/proof-of-sql/Cargo.toml index f0973b312..f11121a68 100644 --- a/crates/proof-of-sql/Cargo.toml +++ b/crates/proof-of-sql/Cargo.toml @@ -121,6 +121,7 @@ required-features = [ "arrow" ] [[example]] name = "avocado-prices" +required-features = [ "arrow" ] [[bench]] name = "posql_benches" diff --git a/crates/proof-of-sql/examples/avocado-prices/main.rs b/crates/proof-of-sql/examples/avocado-prices/main.rs index 44c925049..ea7f3a5ba 100644 --- a/crates/proof-of-sql/examples/avocado-prices/main.rs +++ b/crates/proof-of-sql/examples/avocado-prices/main.rs @@ -1,3 +1,19 @@ //! Example to use Proof of SQL with datasets //! To run, use `cargo run --example avocado-prices`. -fn main() {} +use arrow::datatypes::SchemaRef; +use arrow_csv::{infer_schema_from_files, ReaderBuilder}; +use std::fs::File; + +fn main() { + let filename = "./crates/proof-of-sql/examples/avocado-prices/avocado-prices.csv"; + let data_batch = ReaderBuilder::new(SchemaRef::new( + infer_schema_from_files(&[filename.to_string()], b',', None, true).unwrap(), + )) + .with_header(true) + .build(File::open(filename).unwrap()) + .unwrap() + .next() + .unwrap() + .unwrap(); + println!("{data_batch:?}"); +} From 5199b76348ff8a33b2717edb11f1f5d472b97a6d Mon Sep 17 00:00:00 2001 From: Johnny <132952411+jd-sxt@users.noreply.github.com> Date: Wed, 23 Oct 2024 21:04:27 -0500 Subject: [PATCH 05/10] feat: run SELECT * query in example --- .../examples/avocado-prices/main.rs | 73 ++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/crates/proof-of-sql/examples/avocado-prices/main.rs b/crates/proof-of-sql/examples/avocado-prices/main.rs index ea7f3a5ba..60043d3e0 100644 --- a/crates/proof-of-sql/examples/avocado-prices/main.rs +++ b/crates/proof-of-sql/examples/avocado-prices/main.rs @@ -1,10 +1,41 @@ //! Example to use Proof of SQL with datasets //! To run, use `cargo run --example avocado-prices`. +//! +//! NOTE: If this doesn't work because you do not have the appropriate GPU drivers installed, +//! you can run `cargo run --release --example avocado-prices --no-default-features --features="arrow cpu-perf"` instead. It will be slower for proof generation. use arrow::datatypes::SchemaRef; use arrow_csv::{infer_schema_from_files, ReaderBuilder}; +use proof_of_sql::{ + base::database::{OwnedTable, OwnedTableTestAccessor}, + proof_primitive::dory::{ + DynamicDoryCommitment, DynamicDoryEvaluationProof, ProverSetup, PublicParameters, + VerifierSetup, + }, + sql::{parse::QueryExpr, proof::QueryProof}, +}; +use rand::{rngs::StdRng, SeedableRng}; use std::fs::File; +// We generate the public parameters and the setups used by the prover and verifier for the Dory PCS. +// The `max_nu` should be set such that the maximum table size is less than `2^(2*max_nu-1)`. +// For a sampling: +// max_nu = 3 => max table size is 32 rows +// max_nu = 4 => max table size is 128 rows +// max_nu = 8 => max table size is 32768 rows +// max_nu = 10 => max table size is 0.5 million rows +// max_nu = 15 => max table size is 0.5 billion rows +// max_nu = 20 => max table size is 0.5 trillion rows +// Note: we will eventually load these from a file. +const DORY_SETUP_MAX_NU: usize = 8; +// This should be a "nothing-up-my-sleeve" phrase or number. +const DORY_SEED: [u8; 32] = *b"len 32 rng seed - Space and Time"; + fn main() { + let mut rng = StdRng::from_seed(DORY_SEED); + let public_parameters = PublicParameters::rand(DORY_SETUP_MAX_NU, &mut rng); + let prover_setup = ProverSetup::from(&public_parameters); + let verifier_setup = VerifierSetup::from(&public_parameters); + let filename = "./crates/proof-of-sql/examples/avocado-prices/avocado-prices.csv"; let data_batch = ReaderBuilder::new(SchemaRef::new( infer_schema_from_files(&[filename.to_string()], b',', None, true).unwrap(), @@ -15,5 +46,45 @@ fn main() { .next() .unwrap() .unwrap(); - println!("{data_batch:?}"); + + // Load the table into an "Accessor" so that the prover and verifier can access the data/commitments. + let accessor = OwnedTableTestAccessor::::new_from_table( + "census.income".parse().unwrap(), + OwnedTable::try_from(data_batch).unwrap(), + 0, + &prover_setup, + ); + + // Parse the query: + let query_plan = QueryExpr::::try_new( + "SELECT * FROM income".parse().unwrap(), + "census".parse().unwrap(), + &accessor, + ) + .unwrap(); + + // Generate the proof and result: + print!("Generating proof..."); + let (proof, provable_result) = QueryProof::::new( + query_plan.proof_expr(), + &accessor, + &&prover_setup, + ); + println!("Done."); + + // Verify the result with the proof: + print!("Verifying proof..."); + let result = proof + .verify( + query_plan.proof_expr(), + &accessor, + &provable_result, + &&verifier_setup, + ) + .unwrap(); + println!("Verified."); + + // Display the result + println!("Query Result:"); + println!("{:?}", result.table); } From b504e2d55b00a723a8b66214207889eb66872b4f Mon Sep 17 00:00:00 2001 From: Johnny <132952411+jd-sxt@users.noreply.github.com> Date: Wed, 23 Oct 2024 21:04:28 -0500 Subject: [PATCH 06/10] feat: add more queries to example --- .../examples/avocado-prices/main.rs | 55 +++++++++++++++++-- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/crates/proof-of-sql/examples/avocado-prices/main.rs b/crates/proof-of-sql/examples/avocado-prices/main.rs index 60043d3e0..30cd03365 100644 --- a/crates/proof-of-sql/examples/avocado-prices/main.rs +++ b/crates/proof-of-sql/examples/avocado-prices/main.rs @@ -14,7 +14,7 @@ use proof_of_sql::{ sql::{parse::QueryExpr, proof::QueryProof}, }; use rand::{rngs::StdRng, SeedableRng}; -use std::fs::File; +use std::{fs::File, time::Instant}; // We generate the public parameters and the setups used by the prover and verifier for the Dory PCS. // The `max_nu` should be set such that the maximum table size is less than `2^(2*max_nu-1)`. @@ -30,6 +30,53 @@ const DORY_SETUP_MAX_NU: usize = 8; // This should be a "nothing-up-my-sleeve" phrase or number. const DORY_SEED: [u8; 32] = *b"len 32 rng seed - Space and Time"; +/// # Panics +/// Will panic if the query does not parse or the proof fails to verify. +fn prove_and_verify_query( + sql: &str, + accessor: &OwnedTableTestAccessor, + prover_setup: &ProverSetup, + verifier_setup: &VerifierSetup, +) { + // Parse the query: + println!("Parsing the query: {sql}..."); + let now = Instant::now(); + let query_plan = QueryExpr::::try_new( + sql.parse().unwrap(), + "avocado".parse().unwrap(), + accessor, + ) + .unwrap(); + println!("Done in {} ms.", now.elapsed().as_secs_f64() * 1000.); + + // Generate the proof and result: + print!("Generating proof..."); + let now = Instant::now(); + let (proof, provable_result) = QueryProof::::new( + query_plan.proof_expr(), + accessor, + &prover_setup, + ); + println!("Done in {} ms.", now.elapsed().as_secs_f64() * 1000.); + + // Verify the result with the proof: + print!("Verifying proof..."); + let now = Instant::now(); + let result = proof + .verify( + query_plan.proof_expr(), + accessor, + &provable_result, + &verifier_setup, + ) + .unwrap(); + println!("Verified in {} ms.", now.elapsed().as_secs_f64() * 1000.); + + // Display the result + println!("Query Result:"); + println!("{:?}", result.table); +} + fn main() { let mut rng = StdRng::from_seed(DORY_SEED); let public_parameters = PublicParameters::rand(DORY_SETUP_MAX_NU, &mut rng); @@ -49,7 +96,7 @@ fn main() { // Load the table into an "Accessor" so that the prover and verifier can access the data/commitments. let accessor = OwnedTableTestAccessor::::new_from_table( - "census.income".parse().unwrap(), + "avocado.prices".parse().unwrap(), OwnedTable::try_from(data_batch).unwrap(), 0, &prover_setup, @@ -57,8 +104,8 @@ fn main() { // Parse the query: let query_plan = QueryExpr::::try_new( - "SELECT * FROM income".parse().unwrap(), - "census".parse().unwrap(), + "SELECT * FROM prices".parse().unwrap(), + "avocado".parse().unwrap(), &accessor, ) .unwrap(); From 0410ecd59c4bf15bc7f01a85bd1c5eda65165143 Mon Sep 17 00:00:00 2001 From: Johnny <132952411+jd-sxt@users.noreply.github.com> Date: Wed, 23 Oct 2024 21:04:29 -0500 Subject: [PATCH 07/10] feat: add more queries to example --- .../examples/avocado-prices/main.rs | 38 +++++-------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/crates/proof-of-sql/examples/avocado-prices/main.rs b/crates/proof-of-sql/examples/avocado-prices/main.rs index 30cd03365..97bce93b6 100644 --- a/crates/proof-of-sql/examples/avocado-prices/main.rs +++ b/crates/proof-of-sql/examples/avocado-prices/main.rs @@ -102,36 +102,16 @@ fn main() { &prover_setup, ); - // Parse the query: - let query_plan = QueryExpr::::try_new( - "SELECT * FROM prices".parse().unwrap(), - "avocado".parse().unwrap(), + prove_and_verify_query( + "SELECT COUNT(*) AS total FROM prices", &accessor, - ) - .unwrap(); - - // Generate the proof and result: - print!("Generating proof..."); - let (proof, provable_result) = QueryProof::::new( - query_plan.proof_expr(), + &prover_setup, + &verifier_setup, + ); + prove_and_verify_query( + "SELECT Geography, COUNT(*) AS num_geographies FROM income GROUP BY Geography", &accessor, - &&prover_setup, + &prover_setup, + &verifier_setup, ); - println!("Done."); - - // Verify the result with the proof: - print!("Verifying proof..."); - let result = proof - .verify( - query_plan.proof_expr(), - &accessor, - &provable_result, - &&verifier_setup, - ) - .unwrap(); - println!("Verified."); - - // Display the result - println!("Query Result:"); - println!("{:?}", result.table); } From eca4b1a6aa425ee5764847a9f765f95cf7c4b5a9 Mon Sep 17 00:00:00 2001 From: Johnny <132952411+jd-sxt@users.noreply.github.com> Date: Wed, 23 Oct 2024 21:04:30 -0500 Subject: [PATCH 08/10] feat: add example using postprocessing --- crates/proof-of-sql/examples/avocado-prices/main.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/proof-of-sql/examples/avocado-prices/main.rs b/crates/proof-of-sql/examples/avocado-prices/main.rs index 97bce93b6..a06a332db 100644 --- a/crates/proof-of-sql/examples/avocado-prices/main.rs +++ b/crates/proof-of-sql/examples/avocado-prices/main.rs @@ -11,7 +11,7 @@ use proof_of_sql::{ DynamicDoryCommitment, DynamicDoryEvaluationProof, ProverSetup, PublicParameters, VerifierSetup, }, - sql::{parse::QueryExpr, proof::QueryProof}, + sql::{parse::QueryExpr, postprocessing::apply_postprocessing_steps, proof::QueryProof}, }; use rand::{rngs::StdRng, SeedableRng}; use std::{fs::File, time::Instant}; @@ -70,11 +70,12 @@ fn prove_and_verify_query( &verifier_setup, ) .unwrap(); + let result = apply_postprocessing_steps(result.table, query_plan.postprocessing()); println!("Verified in {} ms.", now.elapsed().as_secs_f64() * 1000.); // Display the result println!("Query Result:"); - println!("{:?}", result.table); + println!("{result:?}"); } fn main() { @@ -109,7 +110,7 @@ fn main() { &verifier_setup, ); prove_and_verify_query( - "SELECT Geography, COUNT(*) AS num_geographies FROM income GROUP BY Geography", + "SELECT Price, COUNT(*) AS total FROM prices GROUP BY Price ORDER BY total", &accessor, &prover_setup, &verifier_setup, From 6274515d9f51e6dec2e4eb3c9bcb0a07176c3d20 Mon Sep 17 00:00:00 2001 From: Johnny <132952411+jd-sxt@users.noreply.github.com> Date: Wed, 23 Oct 2024 21:04:31 -0500 Subject: [PATCH 09/10] feat: add complex query to example --- crates/proof-of-sql/examples/avocado-prices/main.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/proof-of-sql/examples/avocado-prices/main.rs b/crates/proof-of-sql/examples/avocado-prices/main.rs index a06a332db..85d5e50b8 100644 --- a/crates/proof-of-sql/examples/avocado-prices/main.rs +++ b/crates/proof-of-sql/examples/avocado-prices/main.rs @@ -115,4 +115,10 @@ fn main() { &prover_setup, &verifier_setup, ); + prove_and_verify_query( + "SELECT Year, COUNT(*) AS total FROM prices WHERE Price > 100 GROUP BY Year ORDER BY total DESC LIMIT 5", + &accessor, + &prover_setup, + &verifier_setup, + ); } From 97d112d36db8889926a408d622ce31039ad72752 Mon Sep 17 00:00:00 2001 From: Johnny <132952411+jd-sxt@users.noreply.github.com> Date: Wed, 23 Oct 2024 21:04:32 -0500 Subject: [PATCH 10/10] ci: run example in ci --- .github/workflows/lint-and-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 85014b80a..1c3ecadba 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -124,6 +124,8 @@ jobs: run: cargo run --example books - name: Run brands example run: cargo run --example brands + - name: Run avocado-prices example + run: cargo run --example avocado-prices - name: Run plastics example run: cargo run --example plastics - name: Run posql_db example (With Blitzar) @@ -240,4 +242,4 @@ jobs: - name: Install solhint run: npm install -g solhint - name: Run tests - run: solhint -c 'crates/proof-of-sql/.solhint.json' 'crates/proof-of-sql/**/*.sol' -w 0 \ No newline at end of file + run: solhint -c 'crates/proof-of-sql/.solhint.json' 'crates/proof-of-sql/**/*.sol' -w 0