Merge branch 'main' into example-census

crates/proof-of-sql/Cargo.toml

@@ -151,6 +151,10 @@ required-features = [ "arrow" ]
 name = "rockets"
 required-features = [ "arrow" ]
 
+[[example]]
+name = "albums"
+required-features = [ "arrow" ]
+
 [[bench]]
 name = "posql_benches"
 harness = false

crates/proof-of-sql/examples/albums/albums.csv

@@ -0,0 +1,100 @@
+artist,year,genre,album
+Michael Jackson,1982,Pop,Thriller
+The Beatles,1969,Rock,Abbey Road
+Pink Floyd,1973,Progressive Rock,Dark Side of the Moon
+Eagles,1976,Rock,Hotel California
+Fleetwood Mac,1977,Rock,Rumours
+AC/DC,1980,Hard Rock,Back in Black
+Whitney Houston,1992,Pop/R&B,The Bodyguard
+Bee Gees,1977,Disco,Saturday Night Fever
+Queen,1975,Rock,A Night at the Opera
+Bruce Springsteen,1984,Rock,Born in the U.S.A.
+Nirvana,1991,Grunge,Nevermind
+Adele,2011,Pop/Soul,21
+Bob Marley & The Wailers,1977,Reggae,Exodus
+Metallica,1991,Metal,Metallica
+Prince,1984,Pop/Funk,Purple Rain
+Led Zeppelin,1971,Rock,Led Zeppelin IV
+The Rolling Stones,1972,Rock,Exile on Main St.
+David Bowie,1972,Rock,The Rise and Fall of Ziggy Stardust and the Spiders from Mars
+Stevie Wonder,1976,Soul/R&B,Songs in the Key of Life
+Madonna,1984,Pop,Like a Virgin
+Amy Winehouse,2006,Soul/R&B,Back to Black
+Carole King,1971,Folk/Rock,Tapestry
+Dr. Dre,1992,Hip-Hop,The Chronic
+Bruce Springsteen,1975,Rock,Born to Run
+The Beach Boys,1966,Rock,Pet Sounds
+Joni Mitchell,1971,Folk,Blue
+Miles Davis,1959,Jazz,Kind of Blue
+The Clash,1979,Punk Rock,London Calling
+Simon & Garfunkel,1970,Folk Rock,Bridge Over Troubled Water
+Paul Simon,1986,World/Pop,Graceland
+U2,1987,Rock,The Joshua Tree
+Marvin Gaye,1971,Soul/R&B,What's Going On
+Radiohead,1997,Alternative Rock,OK Computer
+The Who,1971,Rock,Who's Next
+Bob Dylan,1965,Folk Rock,Highway 61 Revisited
+Guns N' Roses,1987,Hard Rock,Appetite for Destruction
+The Doors,1967,Rock,The Doors
+Elton John,1973,Rock/Pop,Goodbye Yellow Brick Road
+R.E.M.,1992,Alternative Rock,Automatic for the People
+Kendrick Lamar,2015,Hip-Hop,To Pimp a Butterfly
+The Strokes,2001,Indie Rock,Is This It
+Kanye West,2010,Hip-Hop,My Beautiful Dark Twisted Fantasy
+Beyoncé,2016,R&B/Pop,Lemonade
+Arcade Fire,2004,Indie Rock,Funeral
+Oasis,1995,Britpop,(What's the Story) Morning Glory?
+Daft Punk,2001,Electronic,Discovery
+Nas,1994,Hip-Hop,Illmatic
+Green Day,1994,Punk Rock,Dookie
+Jay-Z,2001,Hip-Hop,The Blueprint
+Taylor Swift,2014,Pop,1989
+Arctic Monkeys,2013,Alternative Rock,AM
+The Weeknd,2020,Pop/R&B,After Hours
+Lana Del Rey,2012,Alternative/Pop,Born to Die
+Tame Impala,2015,Psychedelic Rock,Currents
+Frank Ocean,2012,R&B/Soul,Channel Orange
+Coldplay,2002,Alternative Rock,A Rush of Blood to the Head
+Lady Gaga,2011,Pop,Born This Way
+Black Keys,2010,Blues Rock,Brothers
+Ed Sheeran,2014,Pop,x
+Tyler The Creator,2019,Hip-Hop,IGOR
+Billie Eilish,2019,Pop/Alternative,When We All Fall Asleep
+Tool,2019,Progressive Metal,Fear Inoculum
+SZA,2022,R&B/Soul,SOS
+Rosalía,2022,Flamenco Pop,Motomami
+Harry Styles,2022,Pop,Harry's House
+Bad Bunny,2022,Reggaeton/Latin Trap,Un Verano Sin Ti
+Pearl Jam,1991,Grunge,Ten
+Red Hot Chili Peppers,1991,Alternative Rock,Blood Sugar Sex Magik
+Björk,1997,Art Pop,Homogenic
+The Weeknd,2022,Pop/R&B,Dawn FM
+Kendrick Lamar,2022,Hip-Hop,Mr. Morale & the Big Steppers
+Taylor Swift,2022,Pop,Midnights
+Arctic Monkeys,2022,Alternative Rock,The Car
+Beyoncé,2022,Dance/Pop,Renaissance
+Drake,2022,Hip-Hop/R&B,Honestly
+Post Malone,2022,Pop/Hip-Hop,Twelve Carat Toothache
+Florence + The Machine,2022,Art Rock,Dance Fever
+Jack Harlow,2022,Hip-Hop,Come Home the Kids Miss You
+Lizzo,2022,Pop/R&B,Special
+Olivia Rodrigo,2023,Pop/Rock,GUTS
+Lorde,2013,Art Pop,Pure Heroine
+Talking Heads,1980,New Wave,Remain in Light
+The Velvet Underground,1967,Art Rock,The Velvet Underground & Nico
+Kate Bush,1985,Art Pop,Hounds of Love
+Stevie Nicks,1981,Rock,Bella Donna
+Travis Scott,2018,Hip-Hop,Astroworld
+Portishead,1994,Trip Hop,Dummy
+The Smiths,1986,Alternative Rock,The Queen Is Dead
+Calvin Harris,2012,Electronic Dance,18 Months
+Rihanna,2016,Pop/R&B,Anti
+Dua Lipa,2020,Pop,Future Nostalgia
+The Cure,1989,Gothic Rock,Disintegration
+Foo Fighters,1997,Alternative Rock,The Colour and the Shape
+A Tribe Called Quest,1991,Hip-Hop,The Low End Theory
+Massive Attack,1998,Trip Hop,Mezzanine
+Gorillaz,2001,Alternative/Hip-Hop,Gorillaz
+Depeche Mode,1990,Electronic,Violator
+Rage Against The Machine,1992,Rap Metal,Rage Against The Machine
+Joy Division,1979,Post-Punk,Unknown Pleasures

crates/proof-of-sql/examples/albums/main.rs

@@ -0,0 +1,135 @@
+//! This is a non-interactive example of using Proof of SQL with an albums dataset.
+//! To run this, use `cargo run --release --example albums`.
+//!
+//! NOTE: If this doesn't work because you do not have the appropriate GPU drivers installed,
+//! you can run `cargo run --release --example albums --no-default-features --features="arrow cpu-perf"` instead. It will be slower for proof generation.
+
+use arrow::datatypes::SchemaRef;
+use arrow_csv::{infer_schema_from_files, ReaderBuilder};
+use proof_of_sql::{
+    base::database::{
+        arrow_schema_utility::get_posql_compatible_schema, OwnedTable, OwnedTableTestAccessor,
+        TestAccessor,
+    },
+    proof_primitive::dory::{
+        DynamicDoryCommitment, DynamicDoryEvaluationProof, ProverSetup, PublicParameters,
+        VerifierSetup,
+    },
+    sql::{parse::QueryExpr, postprocessing::apply_postprocessing_steps, proof::QueryProof},
+};
+use rand::{rngs::StdRng, SeedableRng};
+use std::{fs::File, time::Instant};
+
+// We generate the public parameters and the setups used by the prover and verifier for the Dory PCS.
+// The `max_nu` should be set such that the maximum table size is less than `2^(2*max_nu-1)`.
+const DORY_SETUP_MAX_NU: usize = 8;
+// This should be a "nothing-up-my-sleeve" phrase or number.
+const DORY_SEED: [u8; 32] = *b"32f7f321c4ab1234d5e6f7a8b9c0d1e2";
+
+/// # Panics
+/// Will panic if the query does not parse or the proof fails to verify.
+fn prove_and_verify_query(
+    sql: &str,
+    accessor: &OwnedTableTestAccessor<DynamicDoryEvaluationProof>,
+    prover_setup: &ProverSetup,
+    verifier_setup: &VerifierSetup,
+) {
+    // Parse the query:
+    println!("Parsing the query: {sql}...");
+    let now = Instant::now();
+    let query_plan = QueryExpr::<DynamicDoryCommitment>::try_new(
+        sql.parse().unwrap(),
+        "albums".parse().unwrap(),
+        accessor,
+    )
+    .unwrap();
+    println!("Done in {} ms.", now.elapsed().as_secs_f64() * 1000.);
+
+    // Generate the proof and result:
+    print!("Generating proof...");
+    let now = Instant::now();
+    let (proof, provable_result) = QueryProof::<DynamicDoryEvaluationProof>::new(
+        query_plan.proof_expr(),
+        accessor,
+        &prover_setup,
+    );
+    println!("Done in {} ms.", now.elapsed().as_secs_f64() * 1000.);
+
+    // Verify the result with the proof:
+    print!("Verifying proof...");
+    let now = Instant::now();
+    let result = proof
+        .verify(
+            query_plan.proof_expr(),
+            accessor,
+            &provable_result,
+            &verifier_setup,
+        )
+        .unwrap();
+    let result = apply_postprocessing_steps(result.table, query_plan.postprocessing());
+    println!("Verified in {} ms.", now.elapsed().as_secs_f64() * 1000.);
+
+    // Display the result
+    println!("Query Result:");
+    println!("{result:?}");
+}
+
+fn main() {
+    let mut rng = StdRng::from_seed(DORY_SEED);
+    let public_parameters = PublicParameters::rand(DORY_SETUP_MAX_NU, &mut rng);
+    let prover_setup = ProverSetup::from(&public_parameters);
+    let verifier_setup = VerifierSetup::from(&public_parameters);
+
+    let filename = "crates/proof-of-sql/examples/albums/albums.csv";
+    let schema = get_posql_compatible_schema(&SchemaRef::new(
+        infer_schema_from_files(&[filename.to_string()], b',', None, true).unwrap(),
+    ));
+    let albums_batch = ReaderBuilder::new(schema)
+        .with_header(true)
+        .build(File::open(filename).unwrap())
+        .unwrap()
+        .next()
+        .unwrap()
+        .unwrap();
+
+    // Load the table into an "Accessor" so that the prover and verifier can access the data/commitments.
+    let mut accessor =
+        OwnedTableTestAccessor::<DynamicDoryEvaluationProof>::new_empty_with_setup(&prover_setup);
+    accessor.add_table(
+        "albums.collection".parse().unwrap(),
+        OwnedTable::try_from(albums_batch).unwrap(),
+        0,
+    );
+
+    // Query 1: Count number of albums by genre
+    prove_and_verify_query(
+        "SELECT genre, COUNT(*) AS album_count FROM albums.collection GROUP BY genre ORDER BY genre",
+        &accessor,
+        &prover_setup,
+        &verifier_setup,
+    );
+
+    // Query 2: Find all albums from the 1970s
+    prove_and_verify_query(
+        "SELECT artist, album, year FROM albums.collection WHERE year >= 1970 AND year < 1980 ORDER BY year",
+        &accessor,
+        &prover_setup,
+        &verifier_setup,
+    );
+
+    // Query 3: Count total number of albums
+    prove_and_verify_query(
+        "SELECT COUNT(*) AS total_albums FROM albums.collection",
+        &accessor,
+        &prover_setup,
+        &verifier_setup,
+    );
+
+    // Query 4: List all rock albums after 1975 (using exact matches for Rock genres)
+    prove_and_verify_query(
+        "SELECT artist, album, year FROM albums.collection WHERE (genre = 'Rock' OR genre = 'Hard Rock' OR genre = 'Progressive Rock') AND year > 1975 ORDER BY year DESC",
+        &accessor,
+        &prover_setup,
+        &verifier_setup,
+    );
+}