From 69216e40c6663410392f601791a2fb627158a55c Mon Sep 17 00:00:00 2001 From: aw-sxt Date: Thu, 24 Oct 2024 19:59:32 -0500 Subject: [PATCH 01/11] feat: add sushi example stub --- crates/proof-of-sql/Cargo.toml | 4 ++++ crates/proof-of-sql/examples/sushi/main.rs | 6 ++++++ 2 files changed, 10 insertions(+) create mode 100644 crates/proof-of-sql/examples/sushi/main.rs diff --git a/crates/proof-of-sql/Cargo.toml b/crates/proof-of-sql/Cargo.toml index f11121a68..cfa73da37 100644 --- a/crates/proof-of-sql/Cargo.toml +++ b/crates/proof-of-sql/Cargo.toml @@ -123,6 +123,10 @@ required-features = [ "arrow" ] name = "avocado-prices" required-features = [ "arrow" ] +[[example]] +name = "sushi" +required-features = [ "arrow" ] + [[bench]] name = "posql_benches" harness = false diff --git a/crates/proof-of-sql/examples/sushi/main.rs b/crates/proof-of-sql/examples/sushi/main.rs new file mode 100644 index 000000000..c77c93f69 --- /dev/null +++ b/crates/proof-of-sql/examples/sushi/main.rs @@ -0,0 +1,6 @@ +//! This is an non-interactive example of using Proof of SQL with some sushi related datasets. +//! To run this, use `cargo run --example sushi`. + +//! NOTE: If this doesn't work because you do not have the appropriate GPU drivers installed, +//! you can run `cargo run --release --example space --no-default-features --features="arrow cpu-perf"` instead. It will be slower for proof generation. +fn main() {} \ No newline at end of file From 3fce6d3f436a3b957cadf64b2deba0ea78598348 Mon Sep 17 00:00:00 2001 From: aw-sxt Date: Thu, 24 Oct 2024 18:29:19 -0500 Subject: [PATCH 02/11] feat: add dataset for sushi example --- crates/proof-of-sql/examples/sushi/fish.csv | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 crates/proof-of-sql/examples/sushi/fish.csv diff --git a/crates/proof-of-sql/examples/sushi/fish.csv b/crates/proof-of-sql/examples/sushi/fish.csv new file mode 100644 index 000000000..74eb0dbd5 --- /dev/null +++ b/crates/proof-of-sql/examples/sushi/fish.csv @@ -0,0 +1,13 @@ +nameEn,nameJa,kindEn,kindJa,pricePerPound +Tuna,Maguro,Lean Red Meat,Akami,25 +Tuna,Maguro,Medium Fat Read Meat,Toro,65 +Tuna,Maguro,Fatty Red Meat,Otoro,115 +Bonito,Katsuo,Red Meat,Akami,20 +Yellowtail,Hamachi,Red Meat,Akami,27 +Salmon,Salmon,White Fish,Shiromi,17 +Sea Bream,Tai,White Fish,Shiromi,32 +Sea Bass,Suzuki,White Fish,Shiromi,28 +Mackerel,Aji,Silver Skinned,Hikarimono,14 +Sardine,Iwashi,Silver Skinned,Hikarimono,11 +Scallops,Hotate,Shellfish,Kai,26 +Ark-shell clams,Akagai,Shellfish,Kai,29 From d68dce4de202c80689ba8f7c9dcfc48f0a49c995 Mon Sep 17 00:00:00 2001 From: aw-sxt Date: Thu, 24 Oct 2024 18:36:32 -0500 Subject: [PATCH 03/11] feat: load dataset for sushi example --- crates/proof-of-sql/examples/sushi/main.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/crates/proof-of-sql/examples/sushi/main.rs b/crates/proof-of-sql/examples/sushi/main.rs index c77c93f69..69096bba8 100644 --- a/crates/proof-of-sql/examples/sushi/main.rs +++ b/crates/proof-of-sql/examples/sushi/main.rs @@ -3,4 +3,20 @@ //! NOTE: If this doesn't work because you do not have the appropriate GPU drivers installed, //! you can run `cargo run --release --example space --no-default-features --features="arrow cpu-perf"` instead. It will be slower for proof generation. -fn main() {} \ No newline at end of file +use arrow::datatypes::SchemaRef; +use arrow_csv::{infer_schema_from_files, ReaderBuilder}; +use std::fs::File; + +fn main() { + let filename = "./crates/proof-of-sql/examples/sushi/fish.csv"; + let fish_batch = ReaderBuilder::new(SchemaRef::new( + infer_schema_from_files(&[filename.to_string()], b',', None, true).unwrap(), + )) + .with_header(true) + .build(File::open(filename).unwrap()) + .unwrap() + .next() + .unwrap() + .unwrap(); + println!("{fish_batch:?}"); +} \ No newline at end of file From a98cc0469898dd364949febac23f9d601f8bbf76 Mon Sep 17 00:00:00 2001 From: aw-sxt Date: Thu, 24 Oct 2024 18:42:40 -0500 Subject: [PATCH 04/11] feat: setup prover, verifier, & accessor; add function to prove and verify queries --- crates/proof-of-sql/examples/sushi/main.rs | 88 +++++++++++++++++++++- 1 file changed, 86 insertions(+), 2 deletions(-) diff --git a/crates/proof-of-sql/examples/sushi/main.rs b/crates/proof-of-sql/examples/sushi/main.rs index 69096bba8..f0318f719 100644 --- a/crates/proof-of-sql/examples/sushi/main.rs +++ b/crates/proof-of-sql/examples/sushi/main.rs @@ -4,10 +4,90 @@ //! NOTE: If this doesn't work because you do not have the appropriate GPU drivers installed, //! you can run `cargo run --release --example space --no-default-features --features="arrow cpu-perf"` instead. It will be slower for proof generation. use arrow::datatypes::SchemaRef; -use arrow_csv::{infer_schema_from_files, ReaderBuilder}; -use std::fs::File; +use arrow_csv::{ + infer_schema_from_files, + ReaderBuilder +}; +use std::{ + fs::File, + time::Instant +}; +use proof_of_sql::{ + base::database::{ + OwnedTable, + OwnedTableTestAccessor, + TestAccessor + }, + proof_primitive::dory::{ + DynamicDoryCommitment, + DynamicDoryEvaluationProof, + ProverSetup, + PublicParameters, + VerifierSetup + }, + sql::{ + parse::QueryExpr, + proof::QueryProof + } +}; +use rand::{ + rngs::StdRng, + SeedableRng +}; + +const DORY_SETUP_MAX_NU: usize = 8; +const DORY_SEED: [u8; 32] = *b"sushi-is-the-best-food-available"; + +/// # Panics +/// Will panic if the query does not parse or the proof fails to verify. +fn prove_and_verify_query( + sql: &str, + accessor: &OwnedTableTestAccessor, + prover_setup: &ProverSetup, + verifier_setup: &VerifierSetup, +) { + // Parse the query: + println!("Parsing the query: {sql}..."); + let now = Instant::now(); + let query_plan = QueryExpr::::try_new( + sql.parse().unwrap(), + "sushi".parse().unwrap(), + accessor, + ) + .unwrap(); + println!("Done in {} ms.", now.elapsed().as_secs_f64() * 1000.); + // Generate the proof and result: + print!("Generating proof..."); + let now = Instant::now(); + let (proof, provable_result) = QueryProof::::new( + query_plan.proof_expr(), + accessor, + &prover_setup, + ); + println!("Done in {} ms.", now.elapsed().as_secs_f64() * 1000.); + // Verify the result with the proof: + print!("Verifying proof..."); + let now = Instant::now(); + let result = proof + .verify( + query_plan.proof_expr(), + accessor, + &provable_result, + &verifier_setup, + ) + .unwrap(); + println!("Verified in {} ms.", now.elapsed().as_secs_f64() * 1000.); + // Display the result + println!("Query Result:"); + println!("{:?}", result.table); +} fn main() { + let mut rng = StdRng::from_seed(DORY_SEED); + let public_parameters = PublicParameters::rand(DORY_SETUP_MAX_NU, &mut rng); + let prover_setup = ProverSetup::from(&public_parameters); + let verifier_setup = VerifierSetup::from(&public_parameters); + let filename = "./crates/proof-of-sql/examples/sushi/fish.csv"; let fish_batch = ReaderBuilder::new(SchemaRef::new( infer_schema_from_files(&[filename.to_string()], b',', None, true).unwrap(), @@ -19,4 +99,8 @@ fn main() { .unwrap() .unwrap(); println!("{fish_batch:?}"); + + // Load the table into an "Accessor" so that the prover and verifier can access the data/commitments. + let mut accessor = OwnedTableTestAccessor::::new_empty_with_setup(&prover_setup); + accessor.add_table("sushi.fish".parse().unwrap(), OwnedTable::try_from(fish_batch).unwrap(), 0); } \ No newline at end of file From 2ab75b63974b7edf4f74d9c82f7b719be8a9b2f4 Mon Sep 17 00:00:00 2001 From: aw-sxt Date: Thu, 24 Oct 2024 19:03:34 -0500 Subject: [PATCH 05/11] feat: add select * query for sushi example --- crates/proof-of-sql/examples/sushi/main.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/proof-of-sql/examples/sushi/main.rs b/crates/proof-of-sql/examples/sushi/main.rs index f0318f719..c57c6f958 100644 --- a/crates/proof-of-sql/examples/sushi/main.rs +++ b/crates/proof-of-sql/examples/sushi/main.rs @@ -103,4 +103,11 @@ fn main() { // Load the table into an "Accessor" so that the prover and verifier can access the data/commitments. let mut accessor = OwnedTableTestAccessor::::new_empty_with_setup(&prover_setup); accessor.add_table("sushi.fish".parse().unwrap(), OwnedTable::try_from(fish_batch).unwrap(), 0); + + prove_and_verify_query( + "SELECT * FROM fish", + &accessor, + &prover_setup, + &verifier_setup, + ); } \ No newline at end of file From 2dc3b84a6519b5f0cb5c6abea7dc3afc1908ace1 Mon Sep 17 00:00:00 2001 From: aw-sxt Date: Thu, 24 Oct 2024 19:09:06 -0500 Subject: [PATCH 06/11] feat: add tuna queries for sushi example --- crates/proof-of-sql/examples/sushi/main.rs | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/proof-of-sql/examples/sushi/main.rs b/crates/proof-of-sql/examples/sushi/main.rs index c57c6f958..6ab1fb0fe 100644 --- a/crates/proof-of-sql/examples/sushi/main.rs +++ b/crates/proof-of-sql/examples/sushi/main.rs @@ -110,4 +110,27 @@ fn main() { &prover_setup, &verifier_setup, ); + + prove_and_verify_query( + "SELECT COUNT(*) FROM fish WHERE nameEn = 'Tuna'", + &accessor, + &prover_setup, + &verifier_setup, + ); + + prove_and_verify_query( + "SELECT kindEn FROM fish WHERE kindJa = 'Otoro'", + &accessor, + &prover_setup, + &verifier_setup, + ); + + prove_and_verify_query( + "SELECT kindEn FROM fish WHERE kindJa = 'Otoro'", + &accessor, + &prover_setup, + &verifier_setup, + ); + + } \ No newline at end of file From 9998f3de9a4b62bc741854c3b060f6fdd251737c Mon Sep 17 00:00:00 2001 From: aw-sxt Date: Thu, 24 Oct 2024 19:37:06 -0500 Subject: [PATCH 07/11] feat: add price-based query for sushi example --- crates/proof-of-sql/examples/sushi/main.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/proof-of-sql/examples/sushi/main.rs b/crates/proof-of-sql/examples/sushi/main.rs index 6ab1fb0fe..086201e1a 100644 --- a/crates/proof-of-sql/examples/sushi/main.rs +++ b/crates/proof-of-sql/examples/sushi/main.rs @@ -131,6 +131,13 @@ fn main() { &prover_setup, &verifier_setup, ); + + prove_and_verify_query( + "SELECT * FROM fish WHERE pricePerPound > 25 AND pricePerPound < 75", + &accessor, + &prover_setup, + &verifier_setup, + ); } \ No newline at end of file From ad153e318572b49e147dd6b662d611390c54e488 Mon Sep 17 00:00:00 2001 From: aw-sxt Date: Thu, 24 Oct 2024 19:45:28 -0500 Subject: [PATCH 08/11] feat: add group by query for sushi example --- crates/proof-of-sql/examples/sushi/main.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/proof-of-sql/examples/sushi/main.rs b/crates/proof-of-sql/examples/sushi/main.rs index 086201e1a..31dd8e8a3 100644 --- a/crates/proof-of-sql/examples/sushi/main.rs +++ b/crates/proof-of-sql/examples/sushi/main.rs @@ -138,6 +138,11 @@ fn main() { &prover_setup, &verifier_setup, ); - - + + prove_and_verify_query( + "SELECT kindJa, COUNT(*) FROM fish GROUP BY kindJa", + &accessor, + &prover_setup, + &verifier_setup, + ); } \ No newline at end of file From 3c77f19ca61031029d59b879b3a4718ae7f27fb7 Mon Sep 17 00:00:00 2001 From: aw-sxt Date: Thu, 24 Oct 2024 19:49:20 -0500 Subject: [PATCH 09/11] feat: add order by query for sushi example --- crates/proof-of-sql/examples/sushi/main.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/proof-of-sql/examples/sushi/main.rs b/crates/proof-of-sql/examples/sushi/main.rs index 31dd8e8a3..29bc91b30 100644 --- a/crates/proof-of-sql/examples/sushi/main.rs +++ b/crates/proof-of-sql/examples/sushi/main.rs @@ -145,4 +145,11 @@ fn main() { &prover_setup, &verifier_setup, ); + + prove_and_verify_query( + "SELECT kindJa, pricePerPound FROM fish WHERE nameEn = 'Tuna' ORDER BY pricePerPound ASC", + &accessor, + &prover_setup, + &verifier_setup, + ); } \ No newline at end of file From 6d43aaf37b1914c6fc5001e82ea17eeb9b5232a3 Mon Sep 17 00:00:00 2001 From: aw-sxt Date: Thu, 24 Oct 2024 19:50:29 -0500 Subject: [PATCH 10/11] style: run cargo fmt for sushi example --- crates/proof-of-sql/examples/sushi/main.rs | 76 +++++++++------------- 1 file changed, 31 insertions(+), 45 deletions(-) diff --git a/crates/proof-of-sql/examples/sushi/main.rs b/crates/proof-of-sql/examples/sushi/main.rs index 29bc91b30..6fed16ce0 100644 --- a/crates/proof-of-sql/examples/sushi/main.rs +++ b/crates/proof-of-sql/examples/sushi/main.rs @@ -4,36 +4,17 @@ //! NOTE: If this doesn't work because you do not have the appropriate GPU drivers installed, //! you can run `cargo run --release --example space --no-default-features --features="arrow cpu-perf"` instead. It will be slower for proof generation. use arrow::datatypes::SchemaRef; -use arrow_csv::{ - infer_schema_from_files, - ReaderBuilder -}; -use std::{ - fs::File, - time::Instant -}; +use arrow_csv::{infer_schema_from_files, ReaderBuilder}; use proof_of_sql::{ - base::database::{ - OwnedTable, - OwnedTableTestAccessor, - TestAccessor - }, + base::database::{OwnedTable, OwnedTableTestAccessor, TestAccessor}, proof_primitive::dory::{ - DynamicDoryCommitment, - DynamicDoryEvaluationProof, - ProverSetup, - PublicParameters, - VerifierSetup + DynamicDoryCommitment, DynamicDoryEvaluationProof, ProverSetup, PublicParameters, + VerifierSetup, }, - sql::{ - parse::QueryExpr, - proof::QueryProof - } -}; -use rand::{ - rngs::StdRng, - SeedableRng + sql::{parse::QueryExpr, proof::QueryProof}, }; +use rand::{rngs::StdRng, SeedableRng}; +use std::{fs::File, time::Instant}; const DORY_SETUP_MAX_NU: usize = 8; const DORY_SEED: [u8; 32] = *b"sushi-is-the-best-food-available"; @@ -83,73 +64,78 @@ fn prove_and_verify_query( } fn main() { - let mut rng = StdRng::from_seed(DORY_SEED); + let mut rng = StdRng::from_seed(DORY_SEED); let public_parameters = PublicParameters::rand(DORY_SETUP_MAX_NU, &mut rng); let prover_setup = ProverSetup::from(&public_parameters); let verifier_setup = VerifierSetup::from(&public_parameters); - let filename = "./crates/proof-of-sql/examples/sushi/fish.csv"; + let filename = "./crates/proof-of-sql/examples/sushi/fish.csv"; let fish_batch = ReaderBuilder::new(SchemaRef::new( infer_schema_from_files(&[filename.to_string()], b',', None, true).unwrap(), )) - .with_header(true) - .build(File::open(filename).unwrap()) - .unwrap() - .next() - .unwrap() - .unwrap(); + .with_header(true) + .build(File::open(filename).unwrap()) + .unwrap() + .next() + .unwrap() + .unwrap(); println!("{fish_batch:?}"); - // Load the table into an "Accessor" so that the prover and verifier can access the data/commitments. - let mut accessor = OwnedTableTestAccessor::::new_empty_with_setup(&prover_setup); - accessor.add_table("sushi.fish".parse().unwrap(), OwnedTable::try_from(fish_batch).unwrap(), 0); + // Load the table into an "Accessor" so that the prover and verifier can access the data/commitments. + let mut accessor = + OwnedTableTestAccessor::::new_empty_with_setup(&prover_setup); + accessor.add_table( + "sushi.fish".parse().unwrap(), + OwnedTable::try_from(fish_batch).unwrap(), + 0, + ); - prove_and_verify_query( + prove_and_verify_query( "SELECT * FROM fish", &accessor, &prover_setup, &verifier_setup, ); - prove_and_verify_query( + prove_and_verify_query( "SELECT COUNT(*) FROM fish WHERE nameEn = 'Tuna'", &accessor, &prover_setup, &verifier_setup, ); - prove_and_verify_query( + prove_and_verify_query( "SELECT kindEn FROM fish WHERE kindJa = 'Otoro'", &accessor, &prover_setup, &verifier_setup, ); - prove_and_verify_query( + prove_and_verify_query( "SELECT kindEn FROM fish WHERE kindJa = 'Otoro'", &accessor, &prover_setup, &verifier_setup, ); - prove_and_verify_query( + prove_and_verify_query( "SELECT * FROM fish WHERE pricePerPound > 25 AND pricePerPound < 75", &accessor, &prover_setup, &verifier_setup, ); - prove_and_verify_query( + prove_and_verify_query( "SELECT kindJa, COUNT(*) FROM fish GROUP BY kindJa", &accessor, &prover_setup, &verifier_setup, ); - prove_and_verify_query( + prove_and_verify_query( "SELECT kindJa, pricePerPound FROM fish WHERE nameEn = 'Tuna' ORDER BY pricePerPound ASC", &accessor, &prover_setup, &verifier_setup, ); -} \ No newline at end of file +} From 37235f4edded8a2d7372f4b8c51eae804c86bc76 Mon Sep 17 00:00:00 2001 From: aw-sxt Date: Thu, 24 Oct 2024 19:51:26 -0500 Subject: [PATCH 11/11] ci: add sushi example to lint-and-test workflow --- .github/workflows/lint-and-test.yml | 2 ++ crates/proof-of-sql/examples/sushi/fish.csv | 2 +- crates/proof-of-sql/examples/sushi/main.rs | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 1c3ecadba..50d45e760 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -128,6 +128,8 @@ jobs: run: cargo run --example avocado-prices - name: Run plastics example run: cargo run --example plastics + - name: Run sushi example + run: cargo run --example sushi - name: Run posql_db example (With Blitzar) run: bash crates/proof-of-sql/examples/posql_db/run_example.sh - name: Run posql_db example (Without Blitzar) diff --git a/crates/proof-of-sql/examples/sushi/fish.csv b/crates/proof-of-sql/examples/sushi/fish.csv index 74eb0dbd5..e0a14ebc0 100644 --- a/crates/proof-of-sql/examples/sushi/fish.csv +++ b/crates/proof-of-sql/examples/sushi/fish.csv @@ -1,6 +1,6 @@ nameEn,nameJa,kindEn,kindJa,pricePerPound Tuna,Maguro,Lean Red Meat,Akami,25 -Tuna,Maguro,Medium Fat Read Meat,Toro,65 +Tuna,Maguro,Medium Fat Red Meat,Toro,65 Tuna,Maguro,Fatty Red Meat,Otoro,115 Bonito,Katsuo,Red Meat,Akami,20 Yellowtail,Hamachi,Red Meat,Akami,27 diff --git a/crates/proof-of-sql/examples/sushi/main.rs b/crates/proof-of-sql/examples/sushi/main.rs index 6fed16ce0..0c7f89545 100644 --- a/crates/proof-of-sql/examples/sushi/main.rs +++ b/crates/proof-of-sql/examples/sushi/main.rs @@ -2,7 +2,7 @@ //! To run this, use `cargo run --example sushi`. //! NOTE: If this doesn't work because you do not have the appropriate GPU drivers installed, -//! you can run `cargo run --release --example space --no-default-features --features="arrow cpu-perf"` instead. It will be slower for proof generation. +//! you can run `cargo run --release --example sushi --no-default-features --features="arrow cpu-perf"` instead. It will be slower for proof generation. use arrow::datatypes::SchemaRef; use arrow_csv::{infer_schema_from_files, ReaderBuilder}; use proof_of_sql::{