From 11b5f97a97c7644ff8f6039464a632acdd3bc176 Mon Sep 17 00:00:00 2001 From: "dustin.ray" Date: Mon, 28 Oct 2024 16:10:58 -0700 Subject: [PATCH 01/20] feat: generate params utility --- .gitignore | 3 + Cargo.toml | 4 + crates/proof-of-sql/Cargo.toml | 9 + .../utils/generate-parameters/README.md | 69 +++++ .../utils/generate-parameters/main.rs | 264 ++++++++++++++++++ .../generate-parameters/round_trip_test.rs | 57 ++++ 6 files changed, 406 insertions(+) create mode 100644 crates/proof-of-sql/utils/generate-parameters/README.md create mode 100644 crates/proof-of-sql/utils/generate-parameters/main.rs create mode 100644 crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs diff --git a/.gitignore b/.gitignore index eac934387..35fa1e4b5 100644 --- a/.gitignore +++ b/.gitignore @@ -64,3 +64,6 @@ metastore_db # forge test files out cache + +# any output files from generating public params +output/ \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 035636f51..7e6276fc4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ curve25519-dalek = { version = "4", features = ["rand_core"] } derive_more = { version = "0.99" } flexbuffers = { version = "2.0.0" } indexmap = { version = "2.1", default-features = false } +indicatif = "0.17.8" itertools = { version = "0.13.0", default-features = false, features = ["use_alloc"] } lalrpop = { version = "0.22.0" } lalrpop-util = { version = "0.22.0", default-features = false } @@ -48,13 +49,16 @@ postcard = { version = "1.0" } proof-of-sql = { path = "crates/proof-of-sql" } # We automatically update this line during release. So do not modify it! proof-of-sql-parser = { path = "crates/proof-of-sql-parser" } # We automatically update this line during release. So do not modify it! rand = { version = "0.8", default-features = false } +rand_chacha = { version = "0.3.1" } rand_core = { version = "0.6", default-features = false } rayon = { version = "1.5" } serde = { version = "1", default-features = false } serde_json = { version = "1", default-features = false, features = ["alloc"] } +sha2 = "0.10.8" snafu = { version = "0.8.4", default-features = false } sqlparser = { version = "0.45.0", default-features = false } tiny-keccak = { version = "2.0.2", features = [ "keccak" ] } +tempfile = "3.13.0" tracing = { version = "0.1.36", default-features = false } tracing-opentelemetry = { version = "0.22.0" } tracing-subscriber = { version = "0.3.0" } diff --git a/crates/proof-of-sql/Cargo.toml b/crates/proof-of-sql/Cargo.toml index e966aa2bd..92e901deb 100644 --- a/crates/proof-of-sql/Cargo.toml +++ b/crates/proof-of-sql/Cargo.toml @@ -30,10 +30,12 @@ blitzar = { workspace = true, optional = true } bumpalo = { workspace = true, features = ["collections"] } bytemuck = { workspace = true } byte-slice-cast = { workspace = true } +clap = { workspace = true, features = ["derive"] } curve25519-dalek = { workspace = true, features = ["serde"] } chrono = { workspace = true, features = ["serde"] } derive_more = { workspace = true } indexmap = { workspace = true, features = ["serde"] } +indicatif = { workspace = true } itertools = { workspace = true } merlin = { workspace = true, optional = true } num-traits = { workspace = true } @@ -41,9 +43,11 @@ num-bigint = { workspace = true, default-features = false } postcard = { workspace = true, features = ["alloc"] } proof-of-sql-parser = { workspace = true } rand = { workspace = true, default-features = false, optional = true } +rand_chacha = { workspace = true} rayon = { workspace = true, optional = true } serde = { workspace = true, features = ["serde_derive"] } serde_json = { workspace = true } +sha2 = { workspace = true } snafu = { workspace = true } sqlparser = { workspace = true } tiny-keccak = { workspace = true } @@ -62,6 +66,7 @@ opentelemetry-jaeger = { workspace = true } rand = { workspace = true, default-features = false } rand_core = { workspace = true, default-features = false } serde_json = { workspace = true } +tempfile = { workspace = true } tracing = { workspace = true } tracing-opentelemetry = { workspace = true } tracing-subscriber = { workspace = true } @@ -83,6 +88,10 @@ std = ["snafu/std"] [lints] workspace = true +[[bin]] +name = "generate-parameters" +path = "utils/generate-parameters/main.rs" + [[example]] name = "hello_world" required-features = ["test"] diff --git a/crates/proof-of-sql/utils/generate-parameters/README.md b/crates/proof-of-sql/utils/generate-parameters/README.md new file mode 100644 index 000000000..5f2962929 --- /dev/null +++ b/crates/proof-of-sql/utils/generate-parameters/README.md @@ -0,0 +1,69 @@ +# Space and Time ParamGen + +A simple tool to generate the Space and Time public network parameters. + +## 📑 Table of Contents + +- [🛠️ Installation](#installation) +- [📚 Background](#background) +- [🚀 Usage](#usage) +- [📧 Contact](#contact) +- [📚 Additional Resources](#additional-resources) + +## 🛠️ Installation + +Install rust: + +```bash +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +``` + +## 📚 Background + +### What are public parameters? + +There are a wide variety of zero-knowledge proof and argument systems, all offering different performance characterists. The classic example is the Groth16 argument, which establishes a trusted setup (known formally as a common reference string (CRS) or structured reference string (SRS)) to be shared among participants in the network. This setup is structured in such a way that allows arguments of valid computation to be produced with very small sizes. In the case of Groth16, this can be as low as a few group elements or a couple hundred bytes, which is the perfect size to store on a blockchain. + +The Space and Time network makes use of a few different argument systems. The Dory polynomial commitment scheme (PCS) is is a SNARK which requires a setup to be established between the proving and verifying parties. The Dory PCS is chosen because it is ammenable to forming proofs and arguments over matrices, which is perfect for the Proof-Of-SQL case, since databases and tables are essentially matrices. The Dory setup process is unique in that it is *transparent*, meaning there is no toxic waste or secret values to forget once the setup is complete. The setup is initialized with an arbitrary random string which establishes common parameters. We choose the random string "SpaceAndTime" for our setup. This string is a "nothing-up-my-sleeve" number, meaning it is easily auditable and has no hidden structure that can be exploited to generate false proofs or compromise the integrity of the system. + +The Space and Time implementation of the Dory PCS is non-zero knowledge and does not explicty blind the inputs used in the argument of correct sql execution. This yields a leaner implementation and slightly better performance. We may add zero-knowledge blinding in the future, but for now it is not necessary for Proof-Of-SQL to function correctly. + +This tool generates the public setups for either the prover or verifier. The cost of generating these parameters is roughly outlined below. Both setups are parameterized over a value *nu*, which helps establish the maximum size of the table that can be argued against. The prover and the verifer both posses a slightly different setup. The verifier setup is relatively cheap to compute and scales linearly for large nu/table sizes. The prover setup is larger and has a higher cost to compute. We provide pre-computed setups that can easily be downloaded and used with the SxT network in order to skip the expensive generation process, but this repo contains a tool to generate the parameters yourself should you choose to. + +## 🚀 Usage + +Ensure that you have rust installed. Then, clone this repo and simply run the following: + +```bash +cargo run --release +``` + +This generates the setups for both the prover and verifer as two seperate tar.gz files with a default nu value of 14. It saves these parameters at the head of this repo as tar.gz archives. + +You can also just generate the prover setup only: + +```bash +cargo run --release -- --mode p +``` + +Or just the verifier: + +```bash +cargo run --release -- --mode v +``` + +You can also specify a value of nu if needed: + +```bash +cargo run --release -- --mode pv --nu 4 +``` + +## 📧 Contact + +For questions on this repository, please file an issue or reach out to the maintainers and we will reply in earnest. + +## 📚 Additional Resources + +- [Dory: Efficient, Transparent arguments for Generalised Inner Products and Polynomial Commitments](https://eprint.iacr.org/2020/1274) +- [Groth16](https://eprint.iacr.org/2016/260.pdf) +- [Nothing-up-my-sleeve number](https://en.wikipedia.org/wiki/Nothing-up-my-sleeve_number) \ No newline at end of file diff --git a/crates/proof-of-sql/utils/generate-parameters/main.rs b/crates/proof-of-sql/utils/generate-parameters/main.rs new file mode 100644 index 000000000..5ce17ad16 --- /dev/null +++ b/crates/proof-of-sql/utils/generate-parameters/main.rs @@ -0,0 +1,264 @@ +//! doc comment + +#[cfg(test)] +mod round_trip_test; + +use ark_std::rand::SeedableRng; +use clap::Parser; +use indicatif::{ProgressBar, ProgressStyle}; +use proof_of_sql::proof_primitive::dory::{ProverSetup, PublicParameters, VerifierSetup}; +use rand_chacha::ChaCha20Rng; +use sha2::Digest; +use sha2::Sha256; +use std::env; +use std::fs::OpenOptions; +use std::io; +use std::io::Write; +use std::process::Command; +use std::{ + fs::{self, File}, + path::Path, + time::{Duration, Instant}, +}; + +/// Transparent public randomness +const SEED: &str = "SpaceAndTime"; + +// Command-line argument parser structure +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +struct Args { + /// The value for `nu` (number of public parameters) + #[arg(short, long, default_value_t = 8)] + nu: usize, + + /// Mode for generating parameters: "p" for Prover, "v" for Verifier + #[arg(short, long, default_value = "pv")] + mode: String, + + /// The initial randomness for the transparent setup + #[arg(short, long, default_value = SEED)] + seed: String, + + /// The directory to store generated files and archives + #[arg(short, long, default_value = "./output")] + target: String, +} + +fn main() { + // Set the BLITZAR_PARTITION_WINDOW_WIDTH environment variable + env::set_var("BLITZAR_PARTITION_WINDOW_WIDTH", "14"); + + // Confirm that it was set by reading it back + match env::var("BLITZAR_PARTITION_WINDOW_WIDTH") { + Ok(value) => println!( + "Environment variable BLITZAR_PARTITION_WINDOW_WIDTH set to {}", + value + ), + Err(e) => println!("Failed to set environment variable: {}", e), + } + + // Parse command-line arguments + let args = Args::parse(); + + // Ensure the target directory exists + fs::create_dir_all(&args.target).expect("Failed to create target directory"); + + // Clear out the digests.txt file if it already exists + let digests_path = format!("{}/digests_nu_{}.txt", args.target, args.nu); + if Path::new(&digests_path).exists() { + fs::write(&digests_path, "").expect("Failed to clear digests.txt file"); + } + + // Convert the seed string to bytes and create a seeded RNG + let seed_bytes = args + .seed + .bytes() + .chain(std::iter::repeat(0u8)) + .take(32) + .collect::>() + .try_into() + .expect("collection is guaranteed to contain 32 elements"); + let mut rng = ChaCha20Rng::from_seed(seed_bytes); + + let spinner = spinner(format!( + "Generating a random public setup with seed {:?} please wait...", + SEED + )); + + // Use the `nu` value from the command-line argument + let public_parameters = PublicParameters::rand(args.nu, &mut rng); + + spinner.finish_with_message("Public parameter setup complete"); + + match args.mode.as_str() { + "pv" => { + println!("Generating parameters for Prover..."); + generate_prover_setup(&public_parameters, args.nu, &args.target); + println!("Generating parameters for Verifier..."); + generate_verifier_setup(&public_parameters, args.nu, &args.target); + } + "p" => { + println!("Generating parameters for Prover..."); + generate_prover_setup(&public_parameters, args.nu, &args.target); + } + "v" => { + println!("Generating parameters for Verifier..."); + generate_verifier_setup(&public_parameters, args.nu, &args.target); + } + _ => { + println!("Invalid mode! Please choose either 'p' for Prover or 'v' for Verifier."); + } + } +} + +// Generates and writes the ProverSetup from initial public parameters +fn generate_prover_setup(public_parameters: &PublicParameters, nu: usize, target: &str) { + let spinner = spinner( + "Generating parameters for the SxT network. This may take a long time, please wait..." + .into(), + ); + + let start_time = Instant::now(); + + // Heavy operation + let setup = ProverSetup::from(public_parameters); + + spinner.finish_with_message("Prover setup complete."); + let duration = start_time.elapsed(); + println!("Generated prover setup in {:.2?}", duration); + + let public_parameters_path = format!("{}/public_parameters_nu_{}.bin", target, nu); + let result = public_parameters.save_to_file(Path::new(&public_parameters_path)); + let file_path = format!("{}/blitzar_handle_nu_{}.bin", target, nu); + + match result { + Ok(_) => { + write_prover_blitzar_handle(setup, &file_path); + + // Compute and save SHA-256 + let mut digests = Vec::new(); + let public_parameters_digest = compute_sha256(&public_parameters_path); + let blitzar_handle_digest = compute_sha256(&file_path); + if let Some(digest) = public_parameters_digest { + digests.push((public_parameters_path.clone(), digest)); + } + if let Some(digest) = blitzar_handle_digest { + digests.push((file_path.clone(), digest)); + } + save_digests(&digests, target, nu); // Save digests to digests.txt + } + Err(_) => println!("Failed to save parameters, aborting."), + } +} + +// Generates and writes the VerifierSetup from initial public parameters +fn generate_verifier_setup(public_parameters: &PublicParameters, nu: usize, target: &str) { + let spinner = spinner( + "Generating parameters for the SxT network. This may take a long time, please wait..." + .into(), + ); + + let start_time = Instant::now(); + + // Heavy operation + let setup = VerifierSetup::from(public_parameters); + + spinner.finish_with_message("Verifier setup complete."); + let duration = start_time.elapsed(); + println!("Generated verifier setup in {:.2?}", duration); + + let file_path = format!("{}/verifier_setup_nu_{}.bin", target, nu); + let result = write_verifier_setup(setup, &file_path); + + match result { + Ok(_) => { + println!("Verifier setup and parameters saved successfully."); + + // Compute and save SHA-256 + let mut digests = Vec::new(); + if let Some(digest) = compute_sha256(&file_path) { + digests.push((file_path.clone(), digest)); + } + save_digests(&digests, target, nu); // Save digests to digests.txt + } + Err(_) => println!("Failed to save setup, aborting."), + } +} + +// Function to compute SHA-256 hash of a file +fn compute_sha256(file_path: &str) -> Option { + let mut file = File::open(file_path).ok()?; + let mut hasher = Sha256::new(); + io::copy(&mut file, &mut hasher).ok()?; + Some(format!("{:x}", hasher.finalize())) +} + +// Function to save digests to digests.txt +fn save_digests(digests: &[(String, String)], target: &str, nu: usize) { + let digests_path = format!("{}/digests_nu_{}.txt", target, nu); + + // Open file in append mode, creating it if it doesn't exist + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(&digests_path) + .expect("Unable to open or create digests.txt"); + + for (file_path, digest) in digests { + writeln!(file, "{} {}", digest, file_path).expect("Unable to write digest to file"); + } + println!("Digests saved to {}", digests_path); +} + +fn write_prover_blitzar_handle(setup: ProverSetup<'_>, file_path: &str) { + let blitzar_handle = setup.blitzar_handle(); + blitzar_handle.write(file_path); + + // Check the file size to see if it exceeds 2 GB + let metadata = fs::metadata(file_path).expect("Unable to read file metadata"); + let file_size = metadata.len(); + + if file_size > 2 * 1024 * 1024 * 1024 { + // 2 GB in bytes + println!("Handle size exceeds 2 GB, splitting into parts..."); + + // Run `split` command to divide the file into 1.8 GB parts + let split_output = Command::new("split") + .arg("-b") + .arg("1800M") + .arg(file_path) + .arg(&format!("{}.part.", file_path)) + .output() + .expect("Failed to execute split command"); + + if split_output.status.success() { + println!("File successfully split into parts."); + + // Remove the original file after splitting + fs::remove_file(file_path).expect("Failed to remove original blitzar handle file"); + } else { + eprintln!( + "Error during file splitting: {}", + String::from_utf8_lossy(&split_output.stderr) + ); + } + } +} + +fn write_verifier_setup(setup: VerifierSetup, file_path: &str) -> std::io::Result<()> { + setup.save_to_file(Path::new(file_path)) +} + +// Get a spinner so we have haptic feedback during param generation +fn spinner(message: String) -> ProgressBar { + let spinner = ProgressBar::new_spinner(); + spinner.set_style( + ProgressStyle::default_spinner() + .template("{spinner:.green} {msg}") + .unwrap(), + ); + spinner.enable_steady_tick(Duration::from_millis(100)); + spinner.set_message(message); + spinner +} diff --git a/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs b/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs new file mode 100644 index 000000000..f6c086e2c --- /dev/null +++ b/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs @@ -0,0 +1,57 @@ +use proof_of_sql::proof_primitive::dory::{ProverSetup, PublicParameters, VerifierSetup}; +use std::{path::Path, process::Command}; +use tempfile::tempdir; + +#[test] +fn we_can_generate_save_and_load_public_setups() { + // Create a temporary directory for the test + let temp_dir = tempdir().expect("Failed to create a temporary directory"); + let temp_path = temp_dir.path().to_str().unwrap(); + + // Run the binary with nu = 4, mode = "pv", and target as the temp directory + let output = Command::new("cargo") + .arg("run") + .arg("--release") + .arg("--") + .arg("--nu") + .arg("4") + .arg("--mode") + .arg("pv") + .arg("--target") + .arg(temp_path) + .output() + .expect("Failed to execute command"); + + // Check the output to make sure the process ran successfully + assert!( + output.status.success(), + "Process failed to run: {:?}", + output + ); + + // Check that both Prover and Verifier files exist in the temp directory + let blitzar_handle_path = format!("{}/blitzar_handle_nu_4.bin", temp_path); + let verifier_setup_path = format!("{}/verifier_setup_nu_4.bin", temp_path); + let public_parameters_path = format!("{}/public_parameters_nu_4.bin", temp_path); + + assert!( + Path::new(&blitzar_handle_path).exists(), + "Prover setup file is missing" + ); + assert!( + Path::new(&verifier_setup_path).exists(), + "Verifier setup file is missing" + ); + assert!( + Path::new(&public_parameters_path).exists(), + "Public parameters file is missing" + ); + + // Load the ProverSetup and VerifierSetup from their files + let handle = blitzar::compute::MsmHandle::new_from_file(&blitzar_handle_path); + let params = PublicParameters::load_from_file(Path::new(&public_parameters_path)).unwrap(); + + let _prover_setup = ProverSetup::from_public_parameters_and_blitzar_handle(¶ms, handle); + let _verifier_setup = VerifierSetup::load_from_file(Path::new(&verifier_setup_path)) + .expect("Failed to load VerifierSetup"); +} From 38b32c0e2a635ad48da5e0e813a465f629b77e03 Mon Sep 17 00:00:00 2001 From: "dustin.ray" Date: Mon, 28 Oct 2024 16:15:53 -0700 Subject: [PATCH 02/20] fix: clippy --- .../utils/generate-parameters/main.rs | 44 +++++++++---------- .../generate-parameters/round_trip_test.rs | 9 ++-- 2 files changed, 24 insertions(+), 29 deletions(-) diff --git a/crates/proof-of-sql/utils/generate-parameters/main.rs b/crates/proof-of-sql/utils/generate-parameters/main.rs index 5ce17ad16..feeff12ff 100644 --- a/crates/proof-of-sql/utils/generate-parameters/main.rs +++ b/crates/proof-of-sql/utils/generate-parameters/main.rs @@ -8,16 +8,14 @@ use clap::Parser; use indicatif::{ProgressBar, ProgressStyle}; use proof_of_sql::proof_primitive::dory::{ProverSetup, PublicParameters, VerifierSetup}; use rand_chacha::ChaCha20Rng; -use sha2::Digest; -use sha2::Sha256; -use std::env; -use std::fs::OpenOptions; -use std::io; -use std::io::Write; -use std::process::Command; +use sha2::{Digest, Sha256}; use std::{ - fs::{self, File}, + env, + fs::{self, File, OpenOptions}, + io, + io::Write, path::Path, + process::Command, time::{Duration, Instant}, }; @@ -52,10 +50,9 @@ fn main() { // Confirm that it was set by reading it back match env::var("BLITZAR_PARTITION_WINDOW_WIDTH") { Ok(value) => println!( - "Environment variable BLITZAR_PARTITION_WINDOW_WIDTH set to {}", - value + "Environment variable BLITZAR_PARTITION_WINDOW_WIDTH set to {value}" ), - Err(e) => println!("Failed to set environment variable: {}", e), + Err(e) => println!("Failed to set environment variable: {e}"), } // Parse command-line arguments @@ -82,8 +79,7 @@ fn main() { let mut rng = ChaCha20Rng::from_seed(seed_bytes); let spinner = spinner(format!( - "Generating a random public setup with seed {:?} please wait...", - SEED + "Generating a random public setup with seed {SEED:?} please wait..." )); // Use the `nu` value from the command-line argument @@ -126,14 +122,14 @@ fn generate_prover_setup(public_parameters: &PublicParameters, nu: usize, target spinner.finish_with_message("Prover setup complete."); let duration = start_time.elapsed(); - println!("Generated prover setup in {:.2?}", duration); + println!("Generated prover setup in {duration:.2?}"); - let public_parameters_path = format!("{}/public_parameters_nu_{}.bin", target, nu); + let public_parameters_path = format!("{target}/public_parameters_nu_{nu}.bin"); let result = public_parameters.save_to_file(Path::new(&public_parameters_path)); - let file_path = format!("{}/blitzar_handle_nu_{}.bin", target, nu); + let file_path = format!("{target}/blitzar_handle_nu_{nu}.bin"); match result { - Ok(_) => { + Ok(()) => { write_prover_blitzar_handle(setup, &file_path); // Compute and save SHA-256 @@ -166,13 +162,13 @@ fn generate_verifier_setup(public_parameters: &PublicParameters, nu: usize, targ spinner.finish_with_message("Verifier setup complete."); let duration = start_time.elapsed(); - println!("Generated verifier setup in {:.2?}", duration); + println!("Generated verifier setup in {duration:.2?}"); - let file_path = format!("{}/verifier_setup_nu_{}.bin", target, nu); + let file_path = format!("{target}/verifier_setup_nu_{nu}.bin"); let result = write_verifier_setup(setup, &file_path); match result { - Ok(_) => { + Ok(()) => { println!("Verifier setup and parameters saved successfully."); // Compute and save SHA-256 @@ -196,7 +192,7 @@ fn compute_sha256(file_path: &str) -> Option { // Function to save digests to digests.txt fn save_digests(digests: &[(String, String)], target: &str, nu: usize) { - let digests_path = format!("{}/digests_nu_{}.txt", target, nu); + let digests_path = format!("{target}/digests_nu_{nu}.txt"); // Open file in append mode, creating it if it doesn't exist let mut file = OpenOptions::new() @@ -206,9 +202,9 @@ fn save_digests(digests: &[(String, String)], target: &str, nu: usize) { .expect("Unable to open or create digests.txt"); for (file_path, digest) in digests { - writeln!(file, "{} {}", digest, file_path).expect("Unable to write digest to file"); + writeln!(file, "{digest} {file_path}").expect("Unable to write digest to file"); } - println!("Digests saved to {}", digests_path); + println!("Digests saved to {digests_path}"); } fn write_prover_blitzar_handle(setup: ProverSetup<'_>, file_path: &str) { @@ -228,7 +224,7 @@ fn write_prover_blitzar_handle(setup: ProverSetup<'_>, file_path: &str) { .arg("-b") .arg("1800M") .arg(file_path) - .arg(&format!("{}.part.", file_path)) + .arg(format!("{file_path}.part.")) .output() .expect("Failed to execute split command"); diff --git a/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs b/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs index f6c086e2c..864d85767 100644 --- a/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs +++ b/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs @@ -25,14 +25,13 @@ fn we_can_generate_save_and_load_public_setups() { // Check the output to make sure the process ran successfully assert!( output.status.success(), - "Process failed to run: {:?}", - output + "Process failed to run: {output:?}" ); // Check that both Prover and Verifier files exist in the temp directory - let blitzar_handle_path = format!("{}/blitzar_handle_nu_4.bin", temp_path); - let verifier_setup_path = format!("{}/verifier_setup_nu_4.bin", temp_path); - let public_parameters_path = format!("{}/public_parameters_nu_4.bin", temp_path); + let blitzar_handle_path = format!("{temp_path}/blitzar_handle_nu_4.bin"); + let verifier_setup_path = format!("{temp_path}/verifier_setup_nu_4.bin"); + let public_parameters_path = format!("{temp_path}/public_parameters_nu_4.bin"); assert!( Path::new(&blitzar_handle_path).exists(), From a668c480775a33b744fb0d65e3ff839484cf63bf Mon Sep 17 00:00:00 2001 From: Dustin Ray <40841027+Dustin-Ray@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:16:19 -0700 Subject: [PATCH 03/20] Update .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 35fa1e4b5..bd3a4853c 100644 --- a/.gitignore +++ b/.gitignore @@ -66,4 +66,4 @@ out cache # any output files from generating public params -output/ \ No newline at end of file +output/ From 8bab494df14fe7936b309ae09d1cefa10c83a991 Mon Sep 17 00:00:00 2001 From: "dustin.ray" Date: Mon, 28 Oct 2024 16:16:53 -0700 Subject: [PATCH 04/20] readme --- crates/proof-of-sql/utils/generate-parameters/README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/crates/proof-of-sql/utils/generate-parameters/README.md b/crates/proof-of-sql/utils/generate-parameters/README.md index 5f2962929..1399d27a6 100644 --- a/crates/proof-of-sql/utils/generate-parameters/README.md +++ b/crates/proof-of-sql/utils/generate-parameters/README.md @@ -58,10 +58,6 @@ You can also specify a value of nu if needed: cargo run --release -- --mode pv --nu 4 ``` -## 📧 Contact - -For questions on this repository, please file an issue or reach out to the maintainers and we will reply in earnest. - ## 📚 Additional Resources - [Dory: Efficient, Transparent arguments for Generalised Inner Products and Polynomial Commitments](https://eprint.iacr.org/2020/1274) From b6f1618c4fea3d9c10d480fc12356fa578334de8 Mon Sep 17 00:00:00 2001 From: "dustin.ray" Date: Mon, 28 Oct 2024 16:20:51 -0700 Subject: [PATCH 05/20] fmt --- crates/proof-of-sql/utils/generate-parameters/main.rs | 4 +--- .../utils/generate-parameters/round_trip_test.rs | 5 +---- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/crates/proof-of-sql/utils/generate-parameters/main.rs b/crates/proof-of-sql/utils/generate-parameters/main.rs index feeff12ff..00f1b745c 100644 --- a/crates/proof-of-sql/utils/generate-parameters/main.rs +++ b/crates/proof-of-sql/utils/generate-parameters/main.rs @@ -49,9 +49,7 @@ fn main() { // Confirm that it was set by reading it back match env::var("BLITZAR_PARTITION_WINDOW_WIDTH") { - Ok(value) => println!( - "Environment variable BLITZAR_PARTITION_WINDOW_WIDTH set to {value}" - ), + Ok(value) => println!("Environment variable BLITZAR_PARTITION_WINDOW_WIDTH set to {value}"), Err(e) => println!("Failed to set environment variable: {e}"), } diff --git a/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs b/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs index 864d85767..4feafe325 100644 --- a/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs +++ b/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs @@ -23,10 +23,7 @@ fn we_can_generate_save_and_load_public_setups() { .expect("Failed to execute command"); // Check the output to make sure the process ran successfully - assert!( - output.status.success(), - "Process failed to run: {output:?}" - ); + assert!(output.status.success(), "Process failed to run: {output:?}"); // Check that both Prover and Verifier files exist in the temp directory let blitzar_handle_path = format!("{temp_path}/blitzar_handle_nu_4.bin"); From a9f46f18a7708b078f1f66b34ca1b8cd31eee375 Mon Sep 17 00:00:00 2001 From: "dustin.ray" Date: Mon, 28 Oct 2024 18:19:25 -0700 Subject: [PATCH 06/20] toml --- crates/proof-of-sql/Cargo.toml | 1 + crates/proof-of-sql/utils/generate-parameters/main.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/proof-of-sql/Cargo.toml b/crates/proof-of-sql/Cargo.toml index 92e901deb..52479dece 100644 --- a/crates/proof-of-sql/Cargo.toml +++ b/crates/proof-of-sql/Cargo.toml @@ -91,6 +91,7 @@ workspace = true [[bin]] name = "generate-parameters" path = "utils/generate-parameters/main.rs" +required-features = [ "std", "blitzar"] [[example]] name = "hello_world" diff --git a/crates/proof-of-sql/utils/generate-parameters/main.rs b/crates/proof-of-sql/utils/generate-parameters/main.rs index 00f1b745c..7c772ff3f 100644 --- a/crates/proof-of-sql/utils/generate-parameters/main.rs +++ b/crates/proof-of-sql/utils/generate-parameters/main.rs @@ -1,4 +1,4 @@ -//! doc comment +//! A CLI utility to generate the public parameters for the prover and verifier #[cfg(test)] mod round_trip_test; From 79ffa6a4a5c58086d96e21a518d3019d2ab393c2 Mon Sep 17 00:00:00 2001 From: "dustin.ray" Date: Mon, 28 Oct 2024 23:56:27 -0700 Subject: [PATCH 07/20] fix: handle panics with error printlns --- .../utils/generate-parameters/main.rs | 113 +++++++++++------- .../generate-parameters/round_trip_test.rs | 2 + 2 files changed, 75 insertions(+), 40 deletions(-) diff --git a/crates/proof-of-sql/utils/generate-parameters/main.rs b/crates/proof-of-sql/utils/generate-parameters/main.rs index 7c772ff3f..d52390830 100644 --- a/crates/proof-of-sql/utils/generate-parameters/main.rs +++ b/crates/proof-of-sql/utils/generate-parameters/main.rs @@ -22,6 +22,8 @@ use std::{ /// Transparent public randomness const SEED: &str = "SpaceAndTime"; +const BLITZAR_PARTITION_WINDOW_WIDTH: &str = "BLITZAR_PARTITION_WINDOW_WIDTH"; + // Command-line argument parser structure #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] @@ -45,20 +47,30 @@ struct Args { fn main() { // Set the BLITZAR_PARTITION_WINDOW_WIDTH environment variable - env::set_var("BLITZAR_PARTITION_WINDOW_WIDTH", "14"); + env::set_var(BLITZAR_PARTITION_WINDOW_WIDTH, "14"); // Confirm that it was set by reading it back - match env::var("BLITZAR_PARTITION_WINDOW_WIDTH") { - Ok(value) => println!("Environment variable BLITZAR_PARTITION_WINDOW_WIDTH set to {value}"), - Err(e) => println!("Failed to set environment variable: {e}"), + match env::var(BLITZAR_PARTITION_WINDOW_WIDTH) { + Ok(value) => { + println!("Environment variable {BLITZAR_PARTITION_WINDOW_WIDTH} set to {value}"); + } + Err(e) => println!("Failed to set {BLITZAR_PARTITION_WINDOW_WIDTH}: {e}"), } // Parse command-line arguments let args = Args::parse(); // Ensure the target directory exists - fs::create_dir_all(&args.target).expect("Failed to create target directory"); + match fs::create_dir_all(&args.target) { + Ok(_) => generate_parameters(args), + Err(_) => eprint!( + "Skipping generation, failed to write or create target directory: {}. Check path and try again.", + args.target, + ), + }; +} +fn generate_parameters(args: Args) { // Clear out the digests.txt file if it already exists let digests_path = format!("{}/digests_nu_{}.txt", args.target, args.nu); if Path::new(&digests_path).exists() { @@ -163,7 +175,7 @@ fn generate_verifier_setup(public_parameters: &PublicParameters, nu: usize, targ println!("Generated verifier setup in {duration:.2?}"); let file_path = format!("{target}/verifier_setup_nu_{nu}.bin"); - let result = write_verifier_setup(setup, &file_path); + let result = write_verifier_setup(&setup, &file_path); match result { Ok(()) => { @@ -188,21 +200,41 @@ fn compute_sha256(file_path: &str) -> Option { Some(format!("{:x}", hasher.finalize())) } -// Function to save digests to digests.txt +/// Function to save digests to a file, or print to console if file saving fails fn save_digests(digests: &[(String, String)], target: &str, nu: usize) { let digests_path = format!("{target}/digests_nu_{nu}.txt"); - // Open file in append mode, creating it if it doesn't exist - let mut file = OpenOptions::new() + // Attempt to open file in append mode, creating it if it doesn't exist + let mut file = if let Ok(f) = OpenOptions::new() .create(true) .append(true) .open(&digests_path) - .expect("Unable to open or create digests.txt"); + { + Some(f) + } else { + println!("Failed to open or create file at {digests_path}. Printing digests to console."); + None + }; for (file_path, digest) in digests { - writeln!(file, "{digest} {file_path}").expect("Unable to write digest to file"); + if let Some(f) = &mut file { + // Attempt to write to file, fall back to printing if it fails + if writeln!(f, "{digest} {file_path}").is_err() { + println!( + "Failed to write to {digests_path}. Printing remaining digests to console." + ); + file = None; // Stop trying to write to the file + } + } + + if file.is_none() { + println!("{digest} {file_path}"); + } + } + + if file.is_some() { + println!("Digests saved to {digests_path}"); } - println!("Digests saved to {digests_path}"); } fn write_prover_blitzar_handle(setup: ProverSetup<'_>, file_path: &str) { @@ -210,37 +242,38 @@ fn write_prover_blitzar_handle(setup: ProverSetup<'_>, file_path: &str) { blitzar_handle.write(file_path); // Check the file size to see if it exceeds 2 GB - let metadata = fs::metadata(file_path).expect("Unable to read file metadata"); - let file_size = metadata.len(); - - if file_size > 2 * 1024 * 1024 * 1024 { - // 2 GB in bytes - println!("Handle size exceeds 2 GB, splitting into parts..."); - - // Run `split` command to divide the file into 1.8 GB parts - let split_output = Command::new("split") - .arg("-b") - .arg("1800M") - .arg(file_path) - .arg(format!("{file_path}.part.")) - .output() - .expect("Failed to execute split command"); - - if split_output.status.success() { - println!("File successfully split into parts."); - - // Remove the original file after splitting - fs::remove_file(file_path).expect("Failed to remove original blitzar handle file"); - } else { - eprintln!( - "Error during file splitting: {}", - String::from_utf8_lossy(&split_output.stderr) - ); + let metadata_res = fs::metadata(file_path); + match metadata_res { + Ok(m) => { + let file_size = m.len(); + + if file_size > 2 * 1024 * 1024 * 1024 { + // 2 GB in bytes + println!("Handle size exceeds 2 GB, splitting into parts..."); + + // Run `split` command to divide the file into 1.8 GB parts + let split_output = Command::new("split") + .arg("-b") + .arg("1800M") + .arg(file_path) + .arg(format!("{file_path}.part.")) + .output(); + + match split_output { + Ok(_) => { + println!("File successfully split into parts."); + fs::remove_file(file_path) + .unwrap_or_else(|_| eprintln!("Error during file splitting")); + } + Err(e) => eprintln!("Error during file splitting: {e}"), + } + } } + Err(e) => eprintln!("Failed to write blitzar_handle to file: {e}"), } } -fn write_verifier_setup(setup: VerifierSetup, file_path: &str) -> std::io::Result<()> { +fn write_verifier_setup(setup: &VerifierSetup, file_path: &str) -> std::io::Result<()> { setup.save_to_file(Path::new(file_path)) } @@ -250,7 +283,7 @@ fn spinner(message: String) -> ProgressBar { spinner.set_style( ProgressStyle::default_spinner() .template("{spinner:.green} {msg}") - .unwrap(), + .unwrap_or_else(|_| ProgressStyle::default_spinner()), ); spinner.enable_steady_tick(Duration::from_millis(100)); spinner.set_message(message); diff --git a/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs b/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs index 4feafe325..9bc380788 100644 --- a/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs +++ b/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs @@ -2,6 +2,8 @@ use proof_of_sql::proof_primitive::dory::{ProverSetup, PublicParameters, Verifie use std::{path::Path, process::Command}; use tempfile::tempdir; +/// # Panics +/// This test will panic if in a number of non-consequential, expected cases. #[test] fn we_can_generate_save_and_load_public_setups() { // Create a temporary directory for the test From 23075b35050e3ef764c75076ef079e950892b07c Mon Sep 17 00:00:00 2001 From: "dustin.ray" Date: Tue, 29 Oct 2024 00:15:26 -0700 Subject: [PATCH 08/20] fix: clippy --- .../utils/generate-parameters/main.rs | 53 +++++++++++-------- .../generate-parameters/round_trip_test.rs | 2 +- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/crates/proof-of-sql/utils/generate-parameters/main.rs b/crates/proof-of-sql/utils/generate-parameters/main.rs index d52390830..53fa2d927 100644 --- a/crates/proof-of-sql/utils/generate-parameters/main.rs +++ b/crates/proof-of-sql/utils/generate-parameters/main.rs @@ -62,7 +62,7 @@ fn main() { // Ensure the target directory exists match fs::create_dir_all(&args.target) { - Ok(_) => generate_parameters(args), + Ok(()) => generate_parameters(&args), Err(_) => eprint!( "Skipping generation, failed to write or create target directory: {}. Check path and try again.", args.target, @@ -70,31 +70,24 @@ fn main() { }; } -fn generate_parameters(args: Args) { +fn generate_parameters(args: &Args) { // Clear out the digests.txt file if it already exists let digests_path = format!("{}/digests_nu_{}.txt", args.target, args.nu); if Path::new(&digests_path).exists() { - fs::write(&digests_path, "").expect("Failed to clear digests.txt file"); + match fs::write(&digests_path, "") { + Ok(()) => {} + Err(e) => eprintln!("Failed to clear digests.txt file: {e}"), + } } - // Convert the seed string to bytes and create a seeded RNG - let seed_bytes = args - .seed - .bytes() - .chain(std::iter::repeat(0u8)) - .take(32) - .collect::>() - .try_into() - .expect("collection is guaranteed to contain 32 elements"); - let mut rng = ChaCha20Rng::from_seed(seed_bytes); + let mut rng = rng_from_seed(args); let spinner = spinner(format!( "Generating a random public setup with seed {SEED:?} please wait..." )); - // Use the `nu` value from the command-line argument + // Obtain public parameter from nu let public_parameters = PublicParameters::rand(args.nu, &mut rng); - spinner.finish_with_message("Public parameter setup complete"); match args.mode.as_str() { @@ -118,7 +111,23 @@ fn generate_parameters(args: Args) { } } -// Generates and writes the ProverSetup from initial public parameters +/// # Panics +/// expects that a [u8; 32] always contains 32 elements, guaranteed not to panic +fn rng_from_seed(args: &Args) -> ChaCha20Rng { + // Convert the seed string to bytes and create a seeded RNG + let seed_bytes = args + .seed + .bytes() + .chain(std::iter::repeat(0u8)) + .take(32) + .collect::>() + .try_into() + .expect("collection is guaranteed to contain 32 elements"); + ChaCha20Rng::from_seed(seed_bytes) +} + +/// Generates and writes the ```ProverSetup``` from initial public parameters + fn generate_prover_setup(public_parameters: &PublicParameters, nu: usize, target: &str) { let spinner = spinner( "Generating parameters for the SxT network. This may take a long time, please wait..." @@ -135,10 +144,10 @@ fn generate_prover_setup(public_parameters: &PublicParameters, nu: usize, target println!("Generated prover setup in {duration:.2?}"); let public_parameters_path = format!("{target}/public_parameters_nu_{nu}.bin"); - let result = public_parameters.save_to_file(Path::new(&public_parameters_path)); + let param_save_result = public_parameters.save_to_file(Path::new(&public_parameters_path)); let file_path = format!("{target}/blitzar_handle_nu_{nu}.bin"); - match result { + match param_save_result { Ok(()) => { write_prover_blitzar_handle(setup, &file_path); @@ -154,7 +163,7 @@ fn generate_prover_setup(public_parameters: &PublicParameters, nu: usize, target } save_digests(&digests, target, nu); // Save digests to digests.txt } - Err(_) => println!("Failed to save parameters, aborting."), + Err(e) => eprintln!("Failed to save prover setup: {e}."), } } @@ -179,7 +188,7 @@ fn generate_verifier_setup(public_parameters: &PublicParameters, nu: usize, targ match result { Ok(()) => { - println!("Verifier setup and parameters saved successfully."); + println!("Verifier setup saved successfully."); // Compute and save SHA-256 let mut digests = Vec::new(); @@ -188,7 +197,7 @@ fn generate_verifier_setup(public_parameters: &PublicParameters, nu: usize, targ } save_digests(&digests, target, nu); // Save digests to digests.txt } - Err(_) => println!("Failed to save setup, aborting."), + Err(e) => eprintln!("Failed to save verifier setup: {e}."), } } @@ -263,7 +272,7 @@ fn write_prover_blitzar_handle(setup: ProverSetup<'_>, file_path: &str) { Ok(_) => { println!("File successfully split into parts."); fs::remove_file(file_path) - .unwrap_or_else(|_| eprintln!("Error during file splitting")); + .unwrap_or_else(|e| eprintln!("Error during file splitting: {e}")); } Err(e) => eprintln!("Error during file splitting: {e}"), } diff --git a/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs b/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs index 9bc380788..62c835ef5 100644 --- a/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs +++ b/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs @@ -3,7 +3,7 @@ use std::{path::Path, process::Command}; use tempfile::tempdir; /// # Panics -/// This test will panic if in a number of non-consequential, expected cases. +/// This test will panic in a number of non-consequential, expected cases. #[test] fn we_can_generate_save_and_load_public_setups() { // Create a temporary directory for the test From a3d02615d3f218de8a3acf7f51cb587b47b6b1e7 Mon Sep 17 00:00:00 2001 From: Dustin Ray <40841027+Dustin-Ray@users.noreply.github.com> Date: Wed, 30 Oct 2024 17:51:04 -0700 Subject: [PATCH 09/20] Update README.md --- crates/proof-of-sql/utils/generate-parameters/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/proof-of-sql/utils/generate-parameters/README.md b/crates/proof-of-sql/utils/generate-parameters/README.md index 1399d27a6..acd37f44b 100644 --- a/crates/proof-of-sql/utils/generate-parameters/README.md +++ b/crates/proof-of-sql/utils/generate-parameters/README.md @@ -28,7 +28,7 @@ The Space and Time network makes use of a few different argument systems. The Do The Space and Time implementation of the Dory PCS is non-zero knowledge and does not explicty blind the inputs used in the argument of correct sql execution. This yields a leaner implementation and slightly better performance. We may add zero-knowledge blinding in the future, but for now it is not necessary for Proof-Of-SQL to function correctly. -This tool generates the public setups for either the prover or verifier. The cost of generating these parameters is roughly outlined below. Both setups are parameterized over a value *nu*, which helps establish the maximum size of the table that can be argued against. The prover and the verifer both posses a slightly different setup. The verifier setup is relatively cheap to compute and scales linearly for large nu/table sizes. The prover setup is larger and has a higher cost to compute. We provide pre-computed setups that can easily be downloaded and used with the SxT network in order to skip the expensive generation process, but this repo contains a tool to generate the parameters yourself should you choose to. +This tool generates the public setups for either the prover or verifier. Both setups are parameterized over a value *nu*, which helps establish the maximum dimension of the table that can be argued against. The prover and the verifer both posses a slightly different setup. The verifier setup is relatively cheap to compute and scales linearly for large nu/table sizes. The prover setup is larger and has a higher cost to compute. We provide pre-computed setups that can easily be downloaded and used with the SxT network in order to skip the expensive generation process, but this repo contains a tool to generate the parameters at your option. ## 🚀 Usage @@ -62,4 +62,4 @@ cargo run --release -- --mode pv --nu 4 - [Dory: Efficient, Transparent arguments for Generalised Inner Products and Polynomial Commitments](https://eprint.iacr.org/2020/1274) - [Groth16](https://eprint.iacr.org/2016/260.pdf) -- [Nothing-up-my-sleeve number](https://en.wikipedia.org/wiki/Nothing-up-my-sleeve_number) \ No newline at end of file +- [Nothing-up-my-sleeve number](https://en.wikipedia.org/wiki/Nothing-up-my-sleeve_number) From 92a6faad4ddb682bfc091e6d8bccdab8400fb866 Mon Sep 17 00:00:00 2001 From: "dustin.ray" Date: Wed, 30 Oct 2024 18:00:52 -0700 Subject: [PATCH 10/20] tests --- .../utils/generate-parameters/main.rs | 1 - .../generate-parameters/round_trip_test.rs | 66 ++++++++++++++++++- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/crates/proof-of-sql/utils/generate-parameters/main.rs b/crates/proof-of-sql/utils/generate-parameters/main.rs index 53fa2d927..e976622f0 100644 --- a/crates/proof-of-sql/utils/generate-parameters/main.rs +++ b/crates/proof-of-sql/utils/generate-parameters/main.rs @@ -127,7 +127,6 @@ fn rng_from_seed(args: &Args) -> ChaCha20Rng { } /// Generates and writes the ```ProverSetup``` from initial public parameters - fn generate_prover_setup(public_parameters: &PublicParameters, nu: usize, target: &str) { let spinner = spinner( "Generating parameters for the SxT network. This may take a long time, please wait..." diff --git a/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs b/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs index 62c835ef5..a50ad367f 100644 --- a/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs +++ b/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs @@ -1,5 +1,11 @@ use proof_of_sql::proof_primitive::dory::{ProverSetup, PublicParameters, VerifierSetup}; -use std::{path::Path, process::Command}; +use sha2::{Digest, Sha256}; +use std::{ + fs::File, + io::{self, BufRead}, + path::Path, + process::Command, +}; use tempfile::tempdir; /// # Panics @@ -31,6 +37,7 @@ fn we_can_generate_save_and_load_public_setups() { let blitzar_handle_path = format!("{temp_path}/blitzar_handle_nu_4.bin"); let verifier_setup_path = format!("{temp_path}/verifier_setup_nu_4.bin"); let public_parameters_path = format!("{temp_path}/public_parameters_nu_4.bin"); + let digests_path = format!("{temp_path}/digests_nu_4.txt"); assert!( Path::new(&blitzar_handle_path).exists(), @@ -44,6 +51,7 @@ fn we_can_generate_save_and_load_public_setups() { Path::new(&public_parameters_path).exists(), "Public parameters file is missing" ); + assert!(Path::new(&digests_path).exists(), "Digests file is missing"); // Load the ProverSetup and VerifierSetup from their files let handle = blitzar::compute::MsmHandle::new_from_file(&blitzar_handle_path); @@ -52,4 +60,60 @@ fn we_can_generate_save_and_load_public_setups() { let _prover_setup = ProverSetup::from_public_parameters_and_blitzar_handle(¶ms, handle); let _verifier_setup = VerifierSetup::load_from_file(Path::new(&verifier_setup_path)) .expect("Failed to load VerifierSetup"); + + // Verify that the digests.txt file contains the correct hash values + let mut expected_digests = Vec::new(); + + // Compute SHA-256 digests for each file + if let Some(digest) = compute_sha256(&public_parameters_path) { + expected_digests.push((public_parameters_path.clone(), digest)); + } + if let Some(digest) = compute_sha256(&blitzar_handle_path) { + expected_digests.push((blitzar_handle_path.clone(), digest)); + } + if let Some(digest) = compute_sha256(&verifier_setup_path) { + expected_digests.push((verifier_setup_path.clone(), digest)); + } + + // Read and parse digests from the file + let actual_digests = read_digests_from_file(&digests_path); + + // Compare expected digests to those read from digests.txt + for (file_path, expected_digest) in &expected_digests { + let actual_digest = actual_digests.get(file_path).expect(&format!( + "Digest for {} not found in digests.txt", + file_path + )); + assert_eq!( + actual_digest, expected_digest, + "Digest mismatch for {}", + file_path + ); + } +} + +/// Compute SHA-256 hash of a file and return it as a hex string. +fn compute_sha256(file_path: &str) -> Option { + let mut file = File::open(file_path).ok()?; + let mut hasher = Sha256::new(); + io::copy(&mut file, &mut hasher).ok()?; + Some(format!("{:x}", hasher.finalize())) +} + +/// Read digests from the digests file and return them as a HashMap. +fn read_digests_from_file(digests_path: &str) -> std::collections::HashMap { + let file = File::open(digests_path).expect("Failed to open digests file"); + let reader = io::BufReader::new(file); + let mut digests = std::collections::HashMap::new(); + + for line in reader.lines() { + let line = line.expect("Failed to read line from digests file"); + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() == 2 { + let digest = parts[0].to_string(); + let file_path = parts[1].to_string(); + digests.insert(file_path, digest); + } + } + digests } From 7015b80f2827d8a78b959aac8515f983558346ed Mon Sep 17 00:00:00 2001 From: "dustin.ray" Date: Wed, 30 Oct 2024 18:13:32 -0700 Subject: [PATCH 11/20] clippy --- .../utils/generate-parameters/round_trip_test.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs b/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs index a50ad367f..9012da81a 100644 --- a/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs +++ b/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs @@ -80,14 +80,10 @@ fn we_can_generate_save_and_load_public_setups() { // Compare expected digests to those read from digests.txt for (file_path, expected_digest) in &expected_digests { - let actual_digest = actual_digests.get(file_path).expect(&format!( - "Digest for {} not found in digests.txt", - file_path - )); + let actual_digest = actual_digests.get(file_path).unwrap_or_else(|| panic!("Digest for {file_path} not found in digests.txt")); assert_eq!( actual_digest, expected_digest, - "Digest mismatch for {}", - file_path + "Digest mismatch for {file_path}" ); } } @@ -100,7 +96,7 @@ fn compute_sha256(file_path: &str) -> Option { Some(format!("{:x}", hasher.finalize())) } -/// Read digests from the digests file and return them as a HashMap. +/// Read digests from the digests file and return them as a `HashMap`. fn read_digests_from_file(digests_path: &str) -> std::collections::HashMap { let file = File::open(digests_path).expect("Failed to open digests file"); let reader = io::BufReader::new(file); From bf28b679588b6e3a713db271e51e8f0a05ef9935 Mon Sep 17 00:00:00 2001 From: "dustin.ray" Date: Wed, 30 Oct 2024 18:17:17 -0700 Subject: [PATCH 12/20] clippy --- .../utils/generate-parameters/round_trip_test.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs b/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs index 9012da81a..0b1a85a32 100644 --- a/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs +++ b/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs @@ -80,7 +80,9 @@ fn we_can_generate_save_and_load_public_setups() { // Compare expected digests to those read from digests.txt for (file_path, expected_digest) in &expected_digests { - let actual_digest = actual_digests.get(file_path).unwrap_or_else(|| panic!("Digest for {file_path} not found in digests.txt")); + let actual_digest = actual_digests + .get(file_path) + .unwrap_or_else(|| panic!("Digest for {file_path} not found in digests.txt")); assert_eq!( actual_digest, expected_digest, "Digest mismatch for {file_path}" @@ -97,6 +99,8 @@ fn compute_sha256(file_path: &str) -> Option { } /// Read digests from the digests file and return them as a `HashMap`. +/// # Panics +/// because it is a test and is allowed to panic fn read_digests_from_file(digests_path: &str) -> std::collections::HashMap { let file = File::open(digests_path).expect("Failed to open digests file"); let reader = io::BufReader::new(file); From f5614014727c60250941ed7aa479dcc3084e67b5 Mon Sep 17 00:00:00 2001 From: "dustin.ray" Date: Thu, 31 Oct 2024 03:45:36 -0700 Subject: [PATCH 13/20] fix: exit on directory failures --- .../utils/generate-parameters/main.rs | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/crates/proof-of-sql/utils/generate-parameters/main.rs b/crates/proof-of-sql/utils/generate-parameters/main.rs index e976622f0..425832bb6 100644 --- a/crates/proof-of-sql/utils/generate-parameters/main.rs +++ b/crates/proof-of-sql/utils/generate-parameters/main.rs @@ -54,19 +54,24 @@ fn main() { Ok(value) => { println!("Environment variable {BLITZAR_PARTITION_WINDOW_WIDTH} set to {value}"); } - Err(e) => println!("Failed to set {BLITZAR_PARTITION_WINDOW_WIDTH}: {e}"), + Err(e) => { + eprintln!("Failed to set {BLITZAR_PARTITION_WINDOW_WIDTH}: {e}"); + std::process::exit(-1) + } } // Parse command-line arguments let args = Args::parse(); // Ensure the target directory exists - match fs::create_dir_all(&args.target) { - Ok(()) => generate_parameters(&args), - Err(_) => eprint!( - "Skipping generation, failed to write or create target directory: {}. Check path and try again.", - args.target, - ), + if let Ok(()) = fs::create_dir_all(&args.target) { + generate_parameters(&args); + } else { + eprintln!( + "Skipping generation, failed to write or create target directory: {}. Check path and try again.", + args.target, + ); + std::process::exit(-1) }; } @@ -259,10 +264,10 @@ fn write_prover_blitzar_handle(setup: ProverSetup<'_>, file_path: &str) { // 2 GB in bytes println!("Handle size exceeds 2 GB, splitting into parts..."); - // Run `split` command to divide the file into 1.8 GB parts + // Run `split` command to divide the file into 2.0 GB parts let split_output = Command::new("split") .arg("-b") - .arg("1800M") + .arg("2000M") .arg(file_path) .arg(format!("{file_path}.part.")) .output(); From ca813cc6b35fdb58c14ca457beea9ca4912e932d Mon Sep 17 00:00:00 2001 From: "dustin.ray" Date: Thu, 31 Oct 2024 16:21:07 -0700 Subject: [PATCH 14/20] fix: enum for mode of operation --- .../utils/generate-parameters/README.md | 30 +++++++----- .../utils/generate-parameters/main.rs | 48 +++++++++++++------ 2 files changed, 51 insertions(+), 27 deletions(-) diff --git a/crates/proof-of-sql/utils/generate-parameters/README.md b/crates/proof-of-sql/utils/generate-parameters/README.md index acd37f44b..e5c5a3753 100644 --- a/crates/proof-of-sql/utils/generate-parameters/README.md +++ b/crates/proof-of-sql/utils/generate-parameters/README.md @@ -40,23 +40,29 @@ cargo run --release This generates the setups for both the prover and verifer as two seperate tar.gz files with a default nu value of 14. It saves these parameters at the head of this repo as tar.gz archives. -You can also just generate the prover setup only: +1. **Run the Prover setup only**: -```bash -cargo run --release -- --mode p -``` + ```bash + cargo run --release --bin generate-parameters -- --mode p + ``` -Or just the verifier: +2. **Run the Verifier setup only**: -```bash -cargo run --release -- --mode v -``` + ```bash + cargo run --release --bin generate-parameters -- --mode v + ``` -You can also specify a value of nu if needed: +3. **Run both Prover and Verifier setups with a custom `nu` value**: -```bash -cargo run --release -- --mode pv --nu 4 -``` + ```bash + cargo run --release --bin generate-parameters -- --mode pv --nu 4 + ``` + +4. **Specify an output directory** (assuming the `--target` argument specifies the output directory): + + ```bash + cargo run --release --bin generate-parameters -- --mode pv --target ./output + ``` ## 📚 Additional Resources diff --git a/crates/proof-of-sql/utils/generate-parameters/main.rs b/crates/proof-of-sql/utils/generate-parameters/main.rs index 425832bb6..a1e7b7c31 100644 --- a/crates/proof-of-sql/utils/generate-parameters/main.rs +++ b/crates/proof-of-sql/utils/generate-parameters/main.rs @@ -4,16 +4,15 @@ mod round_trip_test; use ark_std::rand::SeedableRng; -use clap::Parser; +use clap::{Parser, ValueEnum}; use indicatif::{ProgressBar, ProgressStyle}; use proof_of_sql::proof_primitive::dory::{ProverSetup, PublicParameters, VerifierSetup}; use rand_chacha::ChaCha20Rng; use sha2::{Digest, Sha256}; use std::{ - env, + env, fmt, fs::{self, File, OpenOptions}, - io, - io::Write, + io::{self, Write}, path::Path, process::Command, time::{Duration, Instant}, @@ -24,7 +23,6 @@ const SEED: &str = "SpaceAndTime"; const BLITZAR_PARTITION_WINDOW_WIDTH: &str = "BLITZAR_PARTITION_WINDOW_WIDTH"; -// Command-line argument parser structure #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] struct Args { @@ -32,9 +30,9 @@ struct Args { #[arg(short, long, default_value_t = 8)] nu: usize, - /// Mode for generating parameters: "p" for Prover, "v" for Verifier - #[arg(short, long, default_value = "pv")] - mode: String, + /// Mode for generating parameters: "p" for Prover, "v" for Verifier, pv for both + #[arg(short, long, default_value_t = Mode::PV)] + mode: Mode, /// The initial randomness for the transparent setup #[arg(short, long, default_value = SEED)] @@ -45,6 +43,29 @@ struct Args { target: String, } +// An enum representing possible modes of operation, +// abbreviated to keep clap commands concise, instead +// of requiring the user to type "ProverAndVerifier", +// they type "pv" +#[derive(Debug, Clone, ValueEnum)] +enum Mode { + P, //Prover + V, //verifier + PV, //Both +} + +// Implement Display for Mode +impl fmt::Display for Mode { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mode_str = match self { + Mode::P => "P", + Mode::V => "V", + Mode::PV => "PV", + }; + write!(f, "{mode_str}") + } +} + fn main() { // Set the BLITZAR_PARTITION_WINDOW_WIDTH environment variable env::set_var(BLITZAR_PARTITION_WINDOW_WIDTH, "14"); @@ -95,24 +116,21 @@ fn generate_parameters(args: &Args) { let public_parameters = PublicParameters::rand(args.nu, &mut rng); spinner.finish_with_message("Public parameter setup complete"); - match args.mode.as_str() { - "pv" => { + match args.mode { + Mode::PV => { println!("Generating parameters for Prover..."); generate_prover_setup(&public_parameters, args.nu, &args.target); println!("Generating parameters for Verifier..."); generate_verifier_setup(&public_parameters, args.nu, &args.target); } - "p" => { + Mode::P => { println!("Generating parameters for Prover..."); generate_prover_setup(&public_parameters, args.nu, &args.target); } - "v" => { + Mode::V => { println!("Generating parameters for Verifier..."); generate_verifier_setup(&public_parameters, args.nu, &args.target); } - _ => { - println!("Invalid mode! Please choose either 'p' for Prover or 'v' for Verifier."); - } } } From 4acd9b17b62d14bad848d328b601925e5c24e718 Mon Sep 17 00:00:00 2001 From: "dustin.ray" Date: Thu, 31 Oct 2024 16:35:22 -0700 Subject: [PATCH 15/20] fix: readme and usage info --- .../utils/generate-parameters/README.md | 36 ++++++------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/crates/proof-of-sql/utils/generate-parameters/README.md b/crates/proof-of-sql/utils/generate-parameters/README.md index e5c5a3753..fe0a1544f 100644 --- a/crates/proof-of-sql/utils/generate-parameters/README.md +++ b/crates/proof-of-sql/utils/generate-parameters/README.md @@ -22,11 +22,11 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh ### What are public parameters? -There are a wide variety of zero-knowledge proof and argument systems, all offering different performance characterists. The classic example is the Groth16 argument, which establishes a trusted setup (known formally as a common reference string (CRS) or structured reference string (SRS)) to be shared among participants in the network. This setup is structured in such a way that allows arguments of valid computation to be produced with very small sizes. In the case of Groth16, this can be as low as a few group elements or a couple hundred bytes, which is the perfect size to store on a blockchain. +There are a wide variety of zero-knowledge proof and argument systems, all offering different performance characterists. The classic example is the [Groth16](https://eprint.iacr.org/2016/260.pdf) argument, a commonly used proof system which establishes a trusted setup (known formally as a common reference string (CRS) or structured reference string (SRS)) to be shared among participants in the network. This setup is structured in such a way that allows arguments of valid computation to be produced with very small sizes. In the case of Groth16, this can be as low as a few group elements or a couple hundred bytes, which is the perfect size to store on a blockchain. -The Space and Time network makes use of a few different argument systems. The Dory polynomial commitment scheme (PCS) is is a SNARK which requires a setup to be established between the proving and verifying parties. The Dory PCS is chosen because it is ammenable to forming proofs and arguments over matrices, which is perfect for the Proof-Of-SQL case, since databases and tables are essentially matrices. The Dory setup process is unique in that it is *transparent*, meaning there is no toxic waste or secret values to forget once the setup is complete. The setup is initialized with an arbitrary random string which establishes common parameters. We choose the random string "SpaceAndTime" for our setup. This string is a "nothing-up-my-sleeve" number, meaning it is easily auditable and has no hidden structure that can be exploited to generate false proofs or compromise the integrity of the system. +The Space and Time network makes use of a few different argument systems. The Dory polynomial commitment scheme (PCS) is is a SNARK which requires a setup to be established between the proving and verifying parties. The Dory PCS is chosen because it is ammenable to forming proofs and arguments over matrices, which is perfect for the Proof-Of-SQL case, since databases and tables are essentially matrices. The Dory setup process is unique in that it is *transparent*, meaning there is no toxic waste or secret values to forget once the setup is complete. The setup is initialized with an arbitrary random string which establishes common parameters. We choose the random string "SpaceAndTime" for our setup. This string is a "[Nothing-up-my-sleeve number](https://en.wikipedia.org/wiki/Nothing-up-my-sleeve_number)", meaning it is easily auditable and has no hidden structure that can be exploited to generate false proofs or compromise the integrity of the system. -The Space and Time implementation of the Dory PCS is non-zero knowledge and does not explicty blind the inputs used in the argument of correct sql execution. This yields a leaner implementation and slightly better performance. We may add zero-knowledge blinding in the future, but for now it is not necessary for Proof-Of-SQL to function correctly. +The Space and Time implementation of the [Dory PCS](https://eprint.iacr.org/2020/1274) is non-zero knowledge and does not explicty blind the inputs used in the argument of correct sql execution. This yields a leaner implementation and slightly better performance. We may add zero-knowledge blinding in the future, but for now it is not necessary for Proof-Of-SQL to function correctly. This tool generates the public setups for either the prover or verifier. Both setups are parameterized over a value *nu*, which helps establish the maximum dimension of the table that can be argued against. The prover and the verifer both posses a slightly different setup. The verifier setup is relatively cheap to compute and scales linearly for large nu/table sizes. The prover setup is larger and has a higher cost to compute. We provide pre-computed setups that can easily be downloaded and used with the SxT network in order to skip the expensive generation process, but this repo contains a tool to generate the parameters at your option. @@ -35,34 +35,18 @@ This tool generates the public setups for either the prover or verifier. Both se Ensure that you have rust installed. Then, clone this repo and simply run the following: ```bash -cargo run --release +cargo run --release --bin generate-parameters ``` This generates the setups for both the prover and verifer as two seperate tar.gz files with a default nu value of 14. It saves these parameters at the head of this repo as tar.gz archives. -1. **Run the Prover setup only**: +| Description | Command | +| --------------- | --------------- | +| Run the Prover setup only | ```cargo run --release --bin generate-parameters -- --mode p``` | +|Run the Verifier setup only | ```cargo run --release --bin generate-parameters -- --mode v``` | +| Run both Prover and Verifier setups with a custom nu value | ```cargo run --release --bin generate-parameters -- --mode pv --nu 4``` | +| Specify an output directory (with --target argument) | ```cargo run --release --bin generate-parameters -- --mode pv --target ./output ``` | - ```bash - cargo run --release --bin generate-parameters -- --mode p - ``` - -2. **Run the Verifier setup only**: - - ```bash - cargo run --release --bin generate-parameters -- --mode v - ``` - -3. **Run both Prover and Verifier setups with a custom `nu` value**: - - ```bash - cargo run --release --bin generate-parameters -- --mode pv --nu 4 - ``` - -4. **Specify an output directory** (assuming the `--target` argument specifies the output directory): - - ```bash - cargo run --release --bin generate-parameters -- --mode pv --target ./output - ``` ## 📚 Additional Resources From a3b67d9aa4c8cfe430251eb207d669f849399f65 Mon Sep 17 00:00:00 2001 From: "dustin.ray" Date: Thu, 31 Oct 2024 16:37:07 -0700 Subject: [PATCH 16/20] fix: default value for clap --- crates/proof-of-sql/utils/generate-parameters/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/proof-of-sql/utils/generate-parameters/main.rs b/crates/proof-of-sql/utils/generate-parameters/main.rs index a1e7b7c31..43a8e8422 100644 --- a/crates/proof-of-sql/utils/generate-parameters/main.rs +++ b/crates/proof-of-sql/utils/generate-parameters/main.rs @@ -31,7 +31,7 @@ struct Args { nu: usize, /// Mode for generating parameters: "p" for Prover, "v" for Verifier, pv for both - #[arg(short, long, default_value_t = Mode::PV)] + #[arg(short, long, default_value = "pv")] mode: Mode, /// The initial randomness for the transparent setup From 7a4ef1a53a8f70d0ad3c9ae2ed045c87fc934375 Mon Sep 17 00:00:00 2001 From: "dustin.ray" Date: Mon, 4 Nov 2024 10:37:51 -0800 Subject: [PATCH 17/20] fix: clap shortcuts --- .../utils/generate-parameters/main.rs | 28 ++++++------------- .../generate-parameters/round_trip_test.rs | 2 +- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/crates/proof-of-sql/utils/generate-parameters/main.rs b/crates/proof-of-sql/utils/generate-parameters/main.rs index 43a8e8422..bfa2af909 100644 --- a/crates/proof-of-sql/utils/generate-parameters/main.rs +++ b/crates/proof-of-sql/utils/generate-parameters/main.rs @@ -10,7 +10,7 @@ use proof_of_sql::proof_primitive::dory::{ProverSetup, PublicParameters, Verifie use rand_chacha::ChaCha20Rng; use sha2::{Digest, Sha256}; use std::{ - env, fmt, + env, fs::{self, File, OpenOptions}, io::{self, Write}, path::Path, @@ -31,7 +31,7 @@ struct Args { nu: usize, /// Mode for generating parameters: "p" for Prover, "v" for Verifier, pv for both - #[arg(short, long, default_value = "pv")] + #[arg(short, long, default_value = "all")] mode: Mode, /// The initial randomness for the transparent setup @@ -49,21 +49,9 @@ struct Args { // they type "pv" #[derive(Debug, Clone, ValueEnum)] enum Mode { - P, //Prover - V, //verifier - PV, //Both -} - -// Implement Display for Mode -impl fmt::Display for Mode { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let mode_str = match self { - Mode::P => "P", - Mode::V => "V", - Mode::PV => "PV", - }; - write!(f, "{mode_str}") - } + Prover, //Prover + Verifier, //verifier + All, //Both } fn main() { @@ -117,17 +105,17 @@ fn generate_parameters(args: &Args) { spinner.finish_with_message("Public parameter setup complete"); match args.mode { - Mode::PV => { + Mode::All => { println!("Generating parameters for Prover..."); generate_prover_setup(&public_parameters, args.nu, &args.target); println!("Generating parameters for Verifier..."); generate_verifier_setup(&public_parameters, args.nu, &args.target); } - Mode::P => { + Mode::Prover => { println!("Generating parameters for Prover..."); generate_prover_setup(&public_parameters, args.nu, &args.target); } - Mode::V => { + Mode::Verifier => { println!("Generating parameters for Verifier..."); generate_verifier_setup(&public_parameters, args.nu, &args.target); } diff --git a/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs b/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs index 0b1a85a32..37aeea7dd 100644 --- a/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs +++ b/crates/proof-of-sql/utils/generate-parameters/round_trip_test.rs @@ -24,7 +24,7 @@ fn we_can_generate_save_and_load_public_setups() { .arg("--nu") .arg("4") .arg("--mode") - .arg("pv") + .arg("all") .arg("--target") .arg(temp_path) .output() From 779e388de510d4a13d9c4108a71e1b7774538482 Mon Sep 17 00:00:00 2001 From: "dustin.ray" Date: Mon, 4 Nov 2024 10:55:51 -0800 Subject: [PATCH 18/20] chore: update readme --- .../utils/generate-parameters/README.md | 49 ++++++++++++------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/crates/proof-of-sql/utils/generate-parameters/README.md b/crates/proof-of-sql/utils/generate-parameters/README.md index fe0a1544f..df495d3ce 100644 --- a/crates/proof-of-sql/utils/generate-parameters/README.md +++ b/crates/proof-of-sql/utils/generate-parameters/README.md @@ -4,20 +4,27 @@ A simple tool to generate the Space and Time public network parameters. ## 📑 Table of Contents -- [🛠️ Installation](#installation) +- [🚀 Quick Start](#quick-start) - [📚 Background](#background) -- [🚀 Usage](#usage) -- [📧 Contact](#contact) - [📚 Additional Resources](#additional-resources) -## 🛠️ Installation +## 🚀 Quick Start -Install rust: +Ensure that you have rust installed. Then, clone this repo and simply run the following: ```bash -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +cargo run --release --bin generate-parameters ``` +This generates the setups for both the prover and verifer as two seperate tar.gz files with a default nu value of 14. It saves these parameters at the head of this repo as tar.gz archives. + +| Description | Command | +| --------------- | --------------- | +| Run the Prover setup only | ```cargo run --release --bin generate-parameters -- --mode p``` | +|Run the Verifier setup only | ```cargo run --release --bin generate-parameters -- --mode v``` | +| Run both Prover and Verifier setups with a custom nu value | ```cargo run --release --bin generate-parameters -- --mode pv --nu 4``` | +| Specify an output directory (with --target argument) | ```cargo run --release --bin generate-parameters -- --mode pv --target ./output ``` | + ## 📚 Background ### What are public parameters? @@ -30,22 +37,28 @@ The Space and Time implementation of the [Dory PCS](https://eprint.iacr.org/2020 This tool generates the public setups for either the prover or verifier. Both setups are parameterized over a value *nu*, which helps establish the maximum dimension of the table that can be argued against. The prover and the verifer both posses a slightly different setup. The verifier setup is relatively cheap to compute and scales linearly for large nu/table sizes. The prover setup is larger and has a higher cost to compute. We provide pre-computed setups that can easily be downloaded and used with the SxT network in order to skip the expensive generation process, but this repo contains a tool to generate the parameters at your option. -## 🚀 Usage +### Table Sizes -Ensure that you have rust installed. Then, clone this repo and simply run the following: +The maximum table supported table size for parameters generated by this tool is determined by the value of max nu. For Dynamic Dory, that size is $2^{2 \nu - 1}$. This translates into the number of rows that the parameter can support below: -```bash -cargo run --release --bin generate-parameters -``` +| ν (nu) | Number of rows | +|--------|-------------------| +| 8 | 33 thousand | +| 9 | 131 thousand | +| 10 | 524 thousand | +| 11 | 2.10 million | +| 12 | 8.39 million | +| 13 | 33.55 million | +| 14 | 134.22 million | +| 15 | 536.87 million | +| 16 | 2.15 billion | +| 17 | 8.59 billion | +| 18 | 34.36 billion | +| 19 | 137.44 billion | +| 20 | 549.76 billion | -This generates the setups for both the prover and verifer as two seperate tar.gz files with a default nu value of 14. It saves these parameters at the head of this repo as tar.gz archives. -| Description | Command | -| --------------- | --------------- | -| Run the Prover setup only | ```cargo run --release --bin generate-parameters -- --mode p``` | -|Run the Verifier setup only | ```cargo run --release --bin generate-parameters -- --mode v``` | -| Run both Prover and Verifier setups with a custom nu value | ```cargo run --release --bin generate-parameters -- --mode pv --nu 4``` | -| Specify an output directory (with --target argument) | ```cargo run --release --bin generate-parameters -- --mode pv --target ./output ``` | +NOTE: Setups using the same random string but different nu values remain compatible with each other up to the minimum value of nu shared between them. SxT as of this writing uses a nu value of 16. Setups with smaller values of nu should be compatible with this setup. ## 📚 Additional Resources From f087ab7367f7f153848e82b20755d04703ecdafb Mon Sep 17 00:00:00 2001 From: "dustin.ray" Date: Mon, 4 Nov 2024 10:59:12 -0800 Subject: [PATCH 19/20] fix: readme --- crates/proof-of-sql/utils/generate-parameters/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/proof-of-sql/utils/generate-parameters/README.md b/crates/proof-of-sql/utils/generate-parameters/README.md index df495d3ce..c98a9efa5 100644 --- a/crates/proof-of-sql/utils/generate-parameters/README.md +++ b/crates/proof-of-sql/utils/generate-parameters/README.md @@ -20,10 +20,10 @@ This generates the setups for both the prover and verifer as two seperate tar.gz | Description | Command | | --------------- | --------------- | -| Run the Prover setup only | ```cargo run --release --bin generate-parameters -- --mode p``` | -|Run the Verifier setup only | ```cargo run --release --bin generate-parameters -- --mode v``` | -| Run both Prover and Verifier setups with a custom nu value | ```cargo run --release --bin generate-parameters -- --mode pv --nu 4``` | -| Specify an output directory (with --target argument) | ```cargo run --release --bin generate-parameters -- --mode pv --target ./output ``` | +| Run the Prover setup only | ```cargo run --release --bin generate-parameters -- --mode prover``` | +|Run the Verifier setup only | ```cargo run --release --bin generate-parameters -- --mode verifier``` | +| Run both Prover and Verifier setups with a custom nu value | ```cargo run --release --bin generate-parameters -- --mode all --nu 4``` | +| Specify an output directory (with --target argument) | ```cargo run --release --bin generate-parameters -- --mode all --target ./output ``` | ## 📚 Background From e435e001b1708cf898a4896df90728351f8aec4d Mon Sep 17 00:00:00 2001 From: "dustin.ray" Date: Tue, 5 Nov 2024 13:17:57 -0800 Subject: [PATCH 20/20] fix: handle eprintln exits --- .../utils/generate-parameters/main.rs | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/crates/proof-of-sql/utils/generate-parameters/main.rs b/crates/proof-of-sql/utils/generate-parameters/main.rs index bfa2af909..c004ecc8c 100644 --- a/crates/proof-of-sql/utils/generate-parameters/main.rs +++ b/crates/proof-of-sql/utils/generate-parameters/main.rs @@ -65,7 +65,6 @@ fn main() { } Err(e) => { eprintln!("Failed to set {BLITZAR_PARTITION_WINDOW_WIDTH}: {e}"); - std::process::exit(-1) } } @@ -90,7 +89,10 @@ fn generate_parameters(args: &Args) { if Path::new(&digests_path).exists() { match fs::write(&digests_path, "") { Ok(()) => {} - Err(e) => eprintln!("Failed to clear digests.txt file: {e}"), + Err(e) => { + eprintln!("Failed to clear digests.txt file: {e}"); + std::process::exit(-1) + } } } @@ -173,7 +175,10 @@ fn generate_prover_setup(public_parameters: &PublicParameters, nu: usize, target } save_digests(&digests, target, nu); // Save digests to digests.txt } - Err(e) => eprintln!("Failed to save prover setup: {e}."), + Err(e) => { + eprintln!("Failed to save prover setup: {e}."); + std::process::exit(-1) + } } } @@ -207,7 +212,10 @@ fn generate_verifier_setup(public_parameters: &PublicParameters, nu: usize, targ } save_digests(&digests, target, nu); // Save digests to digests.txt } - Err(e) => eprintln!("Failed to save verifier setup: {e}."), + Err(e) => { + eprintln!("Failed to save verifier setup: {e}."); + std::process::exit(-1) + } } } @@ -281,14 +289,22 @@ fn write_prover_blitzar_handle(setup: ProverSetup<'_>, file_path: &str) { match split_output { Ok(_) => { println!("File successfully split into parts."); - fs::remove_file(file_path) - .unwrap_or_else(|e| eprintln!("Error during file splitting: {e}")); + fs::remove_file(file_path).unwrap_or_else(|e| { + eprintln!("Error clearing large file during split: {e}"); + std::process::exit(-1) + }); + } + Err(e) => { + eprintln!("Error during file splitting: {e}"); + std::process::exit(-1) } - Err(e) => eprintln!("Error during file splitting: {e}"), } } } - Err(e) => eprintln!("Failed to write blitzar_handle to file: {e}"), + Err(e) => { + eprintln!("Failed to write blitzar_handle to file: {e}"); + std::process::exit(-1) + } } }