From 904b9723307584bd5d40902c0b0352b4a6fd855d Mon Sep 17 00:00:00 2001 From: Venu Vemula Date: Fri, 25 Oct 2024 17:12:47 -0400 Subject: [PATCH 01/11] feat: add countries CSV --- .../examples/countries/countries.csv | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 crates/proof-of-sql/examples/countries/countries.csv diff --git a/crates/proof-of-sql/examples/countries/countries.csv b/crates/proof-of-sql/examples/countries/countries.csv new file mode 100644 index 000000000..ee40560e7 --- /dev/null +++ b/crates/proof-of-sql/examples/countries/countries.csv @@ -0,0 +1,31 @@ +Country,Continent,GDP,GDPP +UnitedStates,NorthAmerica,21137,63543 +China,Asia,14342,10261 +Japan,Asia,5081,40293 +Germany,Europe,3846,46329 +India,Asia,2875,2099 +UnitedKingdom,Europe,2825,42330 +France,Europe,2716,41463 +Italy,Europe,2001,33279 +Brazil,SouthAmerica,1839,8718 +Canada,NorthAmerica,1643,43119 +Russia,EuropeAsia,1637,11229 +SouthKorea,Asia,1622,31489 +Australia,Oceania,1382,53799 +Spain,Europe,1316,28152 +Mexico,NorthAmerica,1265,9958 +Indonesia,Asia,1119,4152 +Netherlands,Europe,902,52477 +SaudiArabia,Asia,793,23206 +Turkey,EuropeAsia,761,9005 +Switzerland,Europe,703,81392 +Argentina,SouthAmerica,449,9921 +Sweden,Europe,528,52073 +Nigeria,Africa,448,2190 +Poland,Europe,594,15673 +Thailand,Asia,509,7306 +SouthAfrica,Africa,350,5883 +Philippines,Asia,402,3685 +Colombia,SouthAmerica,323,6458 +Egypt,Africa,302,3012 +Pakistan,Asia,278,1260 \ No newline at end of file From 552979dbf930262d505b31ceeb2c56cb0676ec24 Mon Sep 17 00:00:00 2001 From: Venu Vemula Date: Fri, 25 Oct 2024 17:14:25 -0400 Subject: [PATCH 02/11] feat: add more countries gdp data --- crates/proof-of-sql/examples/countries/countries.csv | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/proof-of-sql/examples/countries/countries.csv b/crates/proof-of-sql/examples/countries/countries.csv index ee40560e7..7cc4c9cb5 100644 --- a/crates/proof-of-sql/examples/countries/countries.csv +++ b/crates/proof-of-sql/examples/countries/countries.csv @@ -28,4 +28,8 @@ SouthAfrica,Africa,350,5883 Philippines,Asia,402,3685 Colombia,SouthAmerica,323,6458 Egypt,Africa,302,3012 -Pakistan,Asia,278,1260 \ No newline at end of file +Pakistan,Asia,278,1260 +Bangladesh,Asia,302,1855 +Vietnam,Asia,283,2900 +Chile,SouthAmerica,252,13120 +Finland,Europe,268,48888 \ No newline at end of file From ffeec5e30202fec5a242b1187cf0943602f847fa Mon Sep 17 00:00:00 2001 From: Venu Vemula Date: Fri, 25 Oct 2024 17:15:44 -0400 Subject: [PATCH 03/11] feat: correct countries gdp data --- crates/proof-of-sql/examples/countries/countries.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/proof-of-sql/examples/countries/countries.csv b/crates/proof-of-sql/examples/countries/countries.csv index 7cc4c9cb5..397102f8f 100644 --- a/crates/proof-of-sql/examples/countries/countries.csv +++ b/crates/proof-of-sql/examples/countries/countries.csv @@ -28,7 +28,7 @@ SouthAfrica,Africa,350,5883 Philippines,Asia,402,3685 Colombia,SouthAmerica,323,6458 Egypt,Africa,302,3012 -Pakistan,Asia,278,1260 +Pakistan,Asia,278,1450 Bangladesh,Asia,302,1855 Vietnam,Asia,283,2900 Chile,SouthAmerica,252,13120 From f67f8335026ef1d560acb7fca7cd4c51ff3a1e8c Mon Sep 17 00:00:00 2001 From: Venu Vemula Date: Fri, 25 Oct 2024 17:17:16 -0400 Subject: [PATCH 04/11] refactor: rename countries csv to countries_gdp csv --- .../examples/countries/{countries.csv => countries_gdp.csv} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename crates/proof-of-sql/examples/countries/{countries.csv => countries_gdp.csv} (100%) diff --git a/crates/proof-of-sql/examples/countries/countries.csv b/crates/proof-of-sql/examples/countries/countries_gdp.csv similarity index 100% rename from crates/proof-of-sql/examples/countries/countries.csv rename to crates/proof-of-sql/examples/countries/countries_gdp.csv From 8bbfa1c6f4e165007a71ff622cb9b8116f39d543 Mon Sep 17 00:00:00 2001 From: Venu Vemula Date: Fri, 25 Oct 2024 17:47:34 -0400 Subject: [PATCH 05/11] feat: add countries example code --- .../proof-of-sql/examples/countries/main.rs | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 crates/proof-of-sql/examples/countries/main.rs diff --git a/crates/proof-of-sql/examples/countries/main.rs b/crates/proof-of-sql/examples/countries/main.rs new file mode 100644 index 000000000..5e2fc8e29 --- /dev/null +++ b/crates/proof-of-sql/examples/countries/main.rs @@ -0,0 +1,118 @@ +//! This is a non-interactive example of using Proof of SQL with a countries dataset. +//! To run this, use `cargo run --release --example countries`. +//! +//! NOTE: If this doesn't work because you do not have the appropriate GPU drivers installed, +//! you can run `cargo run --release --example countries --no-default-features --features="arrow cpu-perf"` instead. It will be slower for proof generation. + +use arrow::datatypes::SchemaRef; +use arrow_csv::{infer_schema_from_files, ReaderBuilder}; +use proof_of_sql::{ + base::database::{ + arrow_schema_utility::get_posql_compatible_schema, OwnedTable, OwnedTableTestAccessor, + TestAccessor, + }, + proof_primitive::dory::{ + DynamicDoryCommitment, DynamicDoryEvaluationProof, ProverSetup, PublicParameters, + VerifierSetup, + }, + sql::{parse::QueryExpr, postprocessing::apply_postprocessing_steps, proof::QueryProof}, +}; +use rand::{rngs::StdRng, SeedableRng}; +use std::{fs::File, time::Instant}; + +// We generate the public parameters and the setups used by the prover and verifier for the Dory PCS. +// The `max_nu` should be set such that the maximum table size is less than `2^(2*max_nu-1)`. +const DORY_SETUP_MAX_NU: usize = 8; +// This should be a "nothing-up-my-sleeve" phrase or number. +const DORY_SEED: [u8; 32] = *b"7a1b3c8d2e4f9g6h5i0j7k2l8m3n9o1p"; + +/// # Panics +/// Will panic if the query does not parse or the proof fails to verify. +fn prove_and_verify_query( + sql: &str, + accessor: &OwnedTableTestAccessor, + prover_setup: &ProverSetup, + verifier_setup: &VerifierSetup, +) { + // Parse the query: + println!("Parsing the query: {sql}..."); + let now = Instant::now(); + let query_plan = QueryExpr::::try_new( + sql.parse().unwrap(), + "countries".parse().unwrap(), + accessor, + ) + .unwrap(); + println!("Done in {} ms.", now.elapsed().as_secs_f64() * 1000.); + + // Generate the proof and result: + print!("Generating proof..."); + let now = Instant::now(); + let (proof, provable_result) = QueryProof::::new( + query_plan.proof_expr(), + accessor, + &prover_setup, + ); + println!("Done in {} ms.", now.elapsed().as_secs_f64() * 1000.); + + // Verify the result with the proof: + print!("Verifying proof..."); + let now = Instant::now(); + let result = proof + .verify( + query_plan.proof_expr(), + accessor, + &provable_result, + &verifier_setup, + ) + .unwrap(); + let result = apply_postprocessing_steps(result.table, query_plan.postprocessing()); + println!("Verified in {} ms.", now.elapsed().as_secs_f64() * 1000.); + + // Display the result + println!("Query Result:"); + println!("{result:?}"); +} + +fn main() { + let mut rng = StdRng::from_seed(DORY_SEED); + let public_parameters = PublicParameters::rand(DORY_SETUP_MAX_NU, &mut rng); + let prover_setup = ProverSetup::from(&public_parameters); + let verifier_setup = VerifierSetup::from(&public_parameters); + + let filename = "./crates/proof-of-sql/examples/countries/countries_gdp.csv"; + let inferred_schema = + SchemaRef::new(infer_schema_from_files(&[filename.to_string()], b',', None, true).unwrap()); + let posql_compatible_schema = get_posql_compatible_schema(&inferred_schema); + + let countries_batch = ReaderBuilder::new(posql_compatible_schema) + .with_header(true) + .build(File::open(filename).unwrap()) + .unwrap() + .next() + .unwrap() + .unwrap(); + + // Load the table into an "Accessor" so that the prover and verifier can access the data/commitments. + let mut accessor = + OwnedTableTestAccessor::::new_empty_with_setup(&prover_setup); + accessor.add_table( + "countries.countries".parse().unwrap(), + OwnedTable::try_from(countries_batch).unwrap(), + 0, + ); + + prove_and_verify_query( + "SELECT COUNT(*) AS total_countries FROM countries", + &accessor, + &prover_setup, + &verifier_setup, + ); + + prove_and_verify_query( + "SELECT continent, MAX(gdp) as max_gdp, COUNT(*) as country_count FROM countries GROUP BY continent ORDER BY max_gdp DESC", + &accessor, + &prover_setup, + &verifier_setup, + ); +} From 0c284469f84388d2394bdaa03acbf543ebb4e88d Mon Sep 17 00:00:00 2001 From: Venu Vemula Date: Fri, 25 Oct 2024 17:51:45 -0400 Subject: [PATCH 06/11] feat: add example query with filter --- crates/proof-of-sql/examples/countries/main.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/proof-of-sql/examples/countries/main.rs b/crates/proof-of-sql/examples/countries/main.rs index 5e2fc8e29..a17c16c1f 100644 --- a/crates/proof-of-sql/examples/countries/main.rs +++ b/crates/proof-of-sql/examples/countries/main.rs @@ -115,4 +115,11 @@ fn main() { &prover_setup, &verifier_setup, ); + + prove_and_verify_query( + "SELECT country FROM countries WHERE continent = 'Asia'", + &accessor, + &prover_setup, + &verifier_setup, + ); } From 471f64605944ef0644d62cbf1dbc2f844d5533c2 Mon Sep 17 00:00:00 2001 From: Venu Vemula Date: Fri, 25 Oct 2024 17:56:07 -0400 Subject: [PATCH 07/11] feat: add example query with complex filter --- crates/proof-of-sql/examples/countries/main.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/proof-of-sql/examples/countries/main.rs b/crates/proof-of-sql/examples/countries/main.rs index a17c16c1f..5a8244704 100644 --- a/crates/proof-of-sql/examples/countries/main.rs +++ b/crates/proof-of-sql/examples/countries/main.rs @@ -122,4 +122,11 @@ fn main() { &prover_setup, &verifier_setup, ); + + prove_and_verify_query( + "SELECT country FROM countries WHERE gdp > 500 AND gdp < 1500", + &accessor, + &prover_setup, + &verifier_setup, + ); } From 5d106cb2e60ca2824ebddc3d18ff001a5159bff6 Mon Sep 17 00:00:00 2001 From: Venu Vemula Date: Fri, 25 Oct 2024 17:58:43 -0400 Subject: [PATCH 08/11] feat: add countries example to Cargo.toml --- crates/proof-of-sql/Cargo.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/proof-of-sql/Cargo.toml b/crates/proof-of-sql/Cargo.toml index cfa73da37..c2be7d100 100644 --- a/crates/proof-of-sql/Cargo.toml +++ b/crates/proof-of-sql/Cargo.toml @@ -127,6 +127,10 @@ required-features = [ "arrow" ] name = "sushi" required-features = [ "arrow" ] +[[example]] +name = "countries" +required-features = [ "arrow" ] + [[bench]] name = "posql_benches" harness = false From 5dfd6ec780a831518b7ba4eaaee6a9e7857e0170 Mon Sep 17 00:00:00 2001 From: Venu Vemula Date: Fri, 25 Oct 2024 18:01:48 -0400 Subject: [PATCH 09/11] feat: add countries example to lint-and-test.yml --- .github/workflows/lint-and-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 50d45e760..51191d986 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -130,6 +130,8 @@ jobs: run: cargo run --example plastics - name: Run sushi example run: cargo run --example sushi + - name: Run countries example + run: cargo run --example countries - name: Run posql_db example (With Blitzar) run: bash crates/proof-of-sql/examples/posql_db/run_example.sh - name: Run posql_db example (Without Blitzar) From 08e361992c89a088a40366e14936efb8d5a4ee45 Mon Sep 17 00:00:00 2001 From: Venu Vemula Date: Fri, 25 Oct 2024 20:08:33 -0400 Subject: [PATCH 10/11] fix: remove query which genertes wild card --- crates/proof-of-sql/examples/countries/main.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/crates/proof-of-sql/examples/countries/main.rs b/crates/proof-of-sql/examples/countries/main.rs index 5a8244704..520d92775 100644 --- a/crates/proof-of-sql/examples/countries/main.rs +++ b/crates/proof-of-sql/examples/countries/main.rs @@ -109,13 +109,6 @@ fn main() { &verifier_setup, ); - prove_and_verify_query( - "SELECT continent, MAX(gdp) as max_gdp, COUNT(*) as country_count FROM countries GROUP BY continent ORDER BY max_gdp DESC", - &accessor, - &prover_setup, - &verifier_setup, - ); - prove_and_verify_query( "SELECT country FROM countries WHERE continent = 'Asia'", &accessor, From 7d7712f1f06308acba056332682d0100a7b03636 Mon Sep 17 00:00:00 2001 From: Venu Vemula Date: Fri, 25 Oct 2024 20:14:29 -0400 Subject: [PATCH 11/11] refactor: add query for sum --- crates/proof-of-sql/examples/countries/main.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/proof-of-sql/examples/countries/main.rs b/crates/proof-of-sql/examples/countries/main.rs index 520d92775..10bfb8705 100644 --- a/crates/proof-of-sql/examples/countries/main.rs +++ b/crates/proof-of-sql/examples/countries/main.rs @@ -122,4 +122,11 @@ fn main() { &prover_setup, &verifier_setup, ); + + prove_and_verify_query( + "SELECT SUM(gdp) AS total_market_cap FROM countries WHERE country = 'China' OR country = 'India'", + &accessor, + &prover_setup, + &verifier_setup, + ); }