Merge branch 'main' into feat/movie-example

.github/workflows/lint-and-test.yml

@@ -114,15 +114,10 @@ jobs:
         run: cargo run --example hello_world --no-default-features --features="test"
       - name: Run space example
         run: cargo run --example space
-      - name: Run dog breeds example
-        run: cargo run --example dog_breeds
-      - name: Run wood types example
-        run: cargo run --example wood_types
       - name: Run posql_db example (With Blitzar)
         run: bash crates/proof-of-sql/examples/posql_db/run_example.sh
       - name: Run posql_db example (Without Blitzar)
         run: bash crates/proof-of-sql/examples/posql_db/run_example.sh --no-default-features --features="rayon"
-
   clippy:
     name: Clippy
     runs-on: large-8-core-32gb-22-04

Cargo.toml

@@ -20,8 +20,8 @@ ark-poly = { version = "0.4.0" }
 ark-serialize = { version = "0.4.0" }
 ark-std = { version = "0.4.0", default-features = false }
 arrayvec = { version = "0.7", default-features = false }
-arrow = { version = "51.0" }
-arrow-csv = { version = "51.0" }
+arrow = { version = "51.0.0" }
+arrow-csv = { version = "51.0.0" }
 bit-iter = { version = "1.1.1" }
 bigdecimal = { version = "0.4.5", default-features = false, features = ["serde"] }
 blake3 = { version = "1.3.3", default-features = false }

crates/proof-of-sql/Cargo.toml

@@ -107,6 +107,42 @@ required-features = [ "arrow" ]
 name = "movies"
 required-features = [ "arrow" ]
 
+[[example]]
+name = "dinosaurs"
+required-features = [ "arrow" ]
+
+[[example]]
+name = "books"
+required-features = [ "arrow" ]
+
+[[example]]
+name = "programming_books"
+required-features = ["arrow"]
+
+[[example]]
+name = "brands"
+required-features = [ "arrow" ]
+
+[[example]]
+name = "plastics"
+required-features = [ "arrow" ]
+
+[[example]]
+name = "avocado-prices"
+required-features = [ "arrow" ]
+
+[[example]]
+name = "sushi"
+required-features = [ "arrow" ]
+
+[[example]]
+name = "countries"
+required-features = [ "arrow" ]
+
+[[example]]
+name = "rockets"
+required-features = [ "arrow" ]
+
 [[bench]]
 name = "posql_benches"
 harness = false

crates/proof-of-sql/examples/avocado-prices/avocado-prices.csv

@@ -0,0 +1,37 @@
+Year,Price
+1990,96
+1991,100
+1992,269
+1993,149
+1994,127
+1995,153
+1996,232
+1997,127
+1998,249
+1999,240
+2000,241
+2001,90
+2002,91
+2003,169
+2004,167
+2005,56
+2006,230
+2007,174
+2008,124
+2009,92
+2010,201
+2011,167
+2012,125
+2013,147
+2014,285
+2015,154
+2016,106
+2017,223
+2018,85
+2019,145
+2020,147
+2021,68
+2022,142
+2023,281
+2024,164
+

crates/proof-of-sql/examples/avocado-prices/main.rs

@@ -0,0 +1,124 @@
+//! Example to use Proof of SQL with datasets
+//! To run, use `cargo run --example avocado-prices`.
+//!
+//! NOTE: If this doesn't work because you do not have the appropriate GPU drivers installed,
+//! you can run `cargo run --release --example avocado-prices --no-default-features --features="arrow cpu-perf"` instead. It will be slower for proof generation.
+use arrow::datatypes::SchemaRef;
+use arrow_csv::{infer_schema_from_files, ReaderBuilder};
+use proof_of_sql::{
+    base::database::{OwnedTable, OwnedTableTestAccessor},
+    proof_primitive::dory::{
+        DynamicDoryCommitment, DynamicDoryEvaluationProof, ProverSetup, PublicParameters,
+        VerifierSetup,
+    },
+    sql::{parse::QueryExpr, postprocessing::apply_postprocessing_steps, proof::QueryProof},
+};
+use rand::{rngs::StdRng, SeedableRng};
+use std::{fs::File, time::Instant};
+
+// We generate the public parameters and the setups used by the prover and verifier for the Dory PCS.
+// The `max_nu` should be set such that the maximum table size is less than `2^(2*max_nu-1)`.
+// For a sampling:
+// max_nu = 3 => max table size is 32 rows
+// max_nu = 4 => max table size is 128 rows
+// max_nu = 8 => max table size is 32768 rows
+// max_nu = 10 => max table size is 0.5 million rows
+// max_nu = 15 => max table size is 0.5 billion rows
+// max_nu = 20 => max table size is 0.5 trillion rows
+// Note: we will eventually load these from a file.
+const DORY_SETUP_MAX_NU: usize = 8;
+// This should be a "nothing-up-my-sleeve" phrase or number.
+const DORY_SEED: [u8; 32] = *b"len 32 rng seed - Space and Time";
+
+/// # Panics
+/// Will panic if the query does not parse or the proof fails to verify.
+fn prove_and_verify_query(
+    sql: &str,
+    accessor: &OwnedTableTestAccessor<DynamicDoryEvaluationProof>,
+    prover_setup: &ProverSetup,
+    verifier_setup: &VerifierSetup,
+) {
+    // Parse the query:
+    println!("Parsing the query: {sql}...");
+    let now = Instant::now();
+    let query_plan = QueryExpr::<DynamicDoryCommitment>::try_new(
+        sql.parse().unwrap(),
+        "avocado".parse().unwrap(),
+        accessor,
+    )
+    .unwrap();
+    println!("Done in {} ms.", now.elapsed().as_secs_f64() * 1000.);
+
+    // Generate the proof and result:
+    print!("Generating proof...");
+    let now = Instant::now();
+    let (proof, provable_result) = QueryProof::<DynamicDoryEvaluationProof>::new(
+        query_plan.proof_expr(),
+        accessor,
+        &prover_setup,
+    );
+    println!("Done in {} ms.", now.elapsed().as_secs_f64() * 1000.);
+
+    // Verify the result with the proof:
+    print!("Verifying proof...");
+    let now = Instant::now();
+    let result = proof
+        .verify(
+            query_plan.proof_expr(),
+            accessor,
+            &provable_result,
+            &verifier_setup,
+        )
+        .unwrap();
+    let result = apply_postprocessing_steps(result.table, query_plan.postprocessing());
+    println!("Verified in {} ms.", now.elapsed().as_secs_f64() * 1000.);
+
+    // Display the result
+    println!("Query Result:");
+    println!("{result:?}");
+}
+
+fn main() {
+    let mut rng = StdRng::from_seed(DORY_SEED);
+    let public_parameters = PublicParameters::rand(DORY_SETUP_MAX_NU, &mut rng);
+    let prover_setup = ProverSetup::from(&public_parameters);
+    let verifier_setup = VerifierSetup::from(&public_parameters);
+
+    let filename = "./crates/proof-of-sql/examples/avocado-prices/avocado-prices.csv";
+    let data_batch = ReaderBuilder::new(SchemaRef::new(
+        infer_schema_from_files(&[filename.to_string()], b',', None, true).unwrap(),
+    ))
+    .with_header(true)
+    .build(File::open(filename).unwrap())
+    .unwrap()
+    .next()
+    .unwrap()
+    .unwrap();
+
+    // Load the table into an "Accessor" so that the prover and verifier can access the data/commitments.
+    let accessor = OwnedTableTestAccessor::<DynamicDoryEvaluationProof>::new_from_table(
+        "avocado.prices".parse().unwrap(),
+        OwnedTable::try_from(data_batch).unwrap(),
+        0,
+        &prover_setup,
+    );
+
+    prove_and_verify_query(
+        "SELECT COUNT(*) AS total FROM prices",
+        &accessor,
+        &prover_setup,
+        &verifier_setup,
+    );
+    prove_and_verify_query(
+        "SELECT Price, COUNT(*) AS total FROM prices GROUP BY Price ORDER BY total",
+        &accessor,
+        &prover_setup,
+        &verifier_setup,
+    );
+    prove_and_verify_query(
+        "SELECT Year, COUNT(*) AS total FROM prices WHERE Price > 100 GROUP BY Year ORDER BY total DESC LIMIT 5",
+        &accessor,
+        &prover_setup,
+        &verifier_setup,
+    );
+}

crates/proof-of-sql/examples/books/books.csv

@@ -0,0 +1,21 @@
+id,title,author,publication_year,genre,rating
+1,To Kill a Mockingbird,Harper Lee,1960,Fiction,4.5
+2,1984,George Orwell,1949,Science Fiction,4.7
+3,Pride and Prejudice,Jane Austen,1813,Romance,4.3
+4,The Great Gatsby,F. Scott Fitzgerald,1925,Fiction,4.2
+5,The Catcher in the Rye,J.D. Salinger,1951,Fiction,4.0
+6,Moby-Dick,Herman Melville,1851,Adventure,4.1
+7,The Lord of the Rings,J.R.R. Tolkien,1954,Fantasy,4.9
+8,The Hobbit,J.R.R. Tolkien,1937,Fantasy,4.6
+9,Brave New World,Aldous Huxley,1932,Science Fiction,4.4
+10,The Hunger Games,Suzanne Collins,2008,Young Adult,4.3
+11,Harry Potter and the Philosopher's Stone,J.K. Rowling,1997,Fantasy,4.8
+12,The Da Vinci Code,Dan Brown,2003,Thriller,3.9
+13,The Alchemist,Paulo Coelho,1988,Fiction,4.2
+14,The Girl with the Dragon Tattoo,Stieg Larsson,2005,Mystery,4.1
+15,The Hitchhiker's Guide to the Galaxy,Douglas Adams,1979,Science Fiction,4.5
+16,The Shining,Stephen King,1977,Horror,4.3
+17,The Catch-22,Joseph Heller,1961,Satire,4.0
+18,The Chronicles of Narnia,C.S. Lewis,1950,Fantasy,4.7
+19,The Fault in Our Stars,John Green,2012,Young Adult,4.2
+20,The Old Man and the Sea,Ernest Hemingway,1952,Fiction,4.1

crates/proof-of-sql/examples/books/main.rs

@@ -0,0 +1,136 @@
+//! This is a non-interactive example of using Proof of SQL with a books dataset.
+//! To run this, use `cargo run --release --example books`.
+//!
+//! NOTE: If this doesn't work because you do not have the appropriate GPU drivers installed,
+//! you can run `cargo run --release --example books --no-default-features --features="arrow cpu-perf"` instead. It will be slower for proof generation.
+
+use arrow::datatypes::SchemaRef;
+use arrow_csv::{infer_schema_from_files, ReaderBuilder};
+use proof_of_sql::{
+    base::database::{
+        arrow_schema_utility::get_posql_compatible_schema, OwnedTable, OwnedTableTestAccessor,
+        TestAccessor,
+    },
+    proof_primitive::dory::{
+        DynamicDoryCommitment, DynamicDoryEvaluationProof, ProverSetup, PublicParameters,
+        VerifierSetup,
+    },
+    sql::{parse::QueryExpr, postprocessing::apply_postprocessing_steps, proof::QueryProof},
+};
+use rand::{rngs::StdRng, SeedableRng};
+use std::{fs::File, time::Instant};
+
+// We generate the public parameters and the setups used by the prover and verifier for the Dory PCS.
+// The `max_nu` should be set such that the maximum table size is less than `2^(2*max_nu-1)`.
+const DORY_SETUP_MAX_NU: usize = 8;
+// This should be a "nothing-up-my-sleeve" phrase or number.
+const DORY_SEED: [u8; 32] = *b"ebab60d58dee4cc69658939b7c2a582d";
+
+/// # Panics
+/// Will panic if the query does not parse or the proof fails to verify.
+fn prove_and_verify_query(
+    sql: &str,
+    accessor: &OwnedTableTestAccessor<DynamicDoryEvaluationProof>,
+    prover_setup: &ProverSetup,
+    verifier_setup: &VerifierSetup,
+) {
+    // Parse the query:
+    println!("Parsing the query: {sql}...");
+    let now = Instant::now();
+    let query_plan = QueryExpr::<DynamicDoryCommitment>::try_new(
+        sql.parse().unwrap(),
+        "books".parse().unwrap(),
+        accessor,
+    )
+    .unwrap();
+    println!("Done in {} ms.", now.elapsed().as_secs_f64() * 1000.);
+
+    // Generate the proof and result:
+    print!("Generating proof...");
+    let now = Instant::now();
+    let (proof, provable_result) = QueryProof::<DynamicDoryEvaluationProof>::new(
+        query_plan.proof_expr(),
+        accessor,
+        &prover_setup,
+    );
+    println!("Done in {} ms.", now.elapsed().as_secs_f64() * 1000.);
+
+    // Verify the result with the proof:
+    print!("Verifying proof...");
+    let now = Instant::now();
+    let result = proof
+        .verify(
+            query_plan.proof_expr(),
+            accessor,
+            &provable_result,
+            &verifier_setup,
+        )
+        .unwrap();
+    let result = apply_postprocessing_steps(result.table, query_plan.postprocessing());
+    println!("Verified in {} ms.", now.elapsed().as_secs_f64() * 1000.);
+
+    // Display the result
+    println!("Query Result:");
+    println!("{result:?}");
+}
+
+fn main() {
+    let mut rng = StdRng::from_seed(DORY_SEED);
+    let public_parameters = PublicParameters::rand(DORY_SETUP_MAX_NU, &mut rng);
+    let prover_setup = ProverSetup::from(&public_parameters);
+    let verifier_setup = VerifierSetup::from(&public_parameters);
+
+    let filename = "./crates/proof-of-sql/examples/books/books.csv";
+    let inferred_schema =
+        SchemaRef::new(infer_schema_from_files(&[filename.to_string()], b',', None, true).unwrap());
+    let posql_compatible_schema = get_posql_compatible_schema(&inferred_schema);
+
+    let books_batch = ReaderBuilder::new(posql_compatible_schema)
+        .with_header(true)
+        .build(File::open(filename).unwrap())
+        .unwrap()
+        .next()
+        .unwrap()
+        .unwrap();
+
+    // Load the table into an "Accessor" so that the prover and verifier can access the data/commitments.
+    let mut accessor =
+        OwnedTableTestAccessor::<DynamicDoryEvaluationProof>::new_empty_with_setup(&prover_setup);
+    accessor.add_table(
+        "books.books".parse().unwrap(),
+        OwnedTable::try_from(books_batch).unwrap(),
+        0,
+    );
+
+    // Query 1: Count the total number of books
+    prove_and_verify_query(
+        "SELECT COUNT(*) AS total_books FROM books",
+        &accessor,
+        &prover_setup,
+        &verifier_setup,
+    );
+
+    // Query 2: Find the top 5 highest-rated books
+    prove_and_verify_query(
+        "SELECT title, author, rating FROM books ORDER BY rating DESC LIMIT 5",
+        &accessor,
+        &prover_setup,
+        &verifier_setup,
+    );
+
+    // Query 3: Count the number of books in each genre
+    prove_and_verify_query(
+        "SELECT genre, COUNT(*) AS book_count FROM books GROUP BY genre ORDER BY book_count DESC",
+        &accessor,
+        &prover_setup,
+        &verifier_setup,
+    );
+
+    // Query 4: Find books published after 2000 with a rating higher than 4.5
+    prove_and_verify_query(
+        "SELECT title, author, publication_year, rating FROM books WHERE publication_year > 2000 AND rating > 4.5",
+        &accessor,
+        &prover_setup,
+        &verifier_setup,
+    );
+}