From f45e684e3e382c36c612b392e8deb18dd57a0f0f Mon Sep 17 00:00:00 2001 From: Jay White Date: Mon, 29 Jul 2024 21:44:19 -0400 Subject: [PATCH 1/2] feat: add dynamic dory structure methods Adds `row_and_column_from_index` and `row_start_index` methods. These methods define the structure of the new dynamic dory commitmemt scheme. Tests are also added, in part to illustrate the structure. --- .../dory/dynamic_dory_structure.rs | 129 ++++++++++++++++++ .../src/proof_primitive/dory/mod.rs | 2 + 2 files changed, 131 insertions(+) create mode 100644 crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_structure.rs diff --git a/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_structure.rs b/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_structure.rs new file mode 100644 index 000000000..d32d14f4d --- /dev/null +++ b/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_structure.rs @@ -0,0 +1,129 @@ +//! This module gives some utility functions for determining the position of data within the dynamic dory matrix +//! +//! In general, the data is filled in such a way that the new data is always in the last row, and the row size +//! (and consequently, the matrix size) is strictly increasing. +//! Aside from the first 3 rows, the pattern is to have 3\*2^n rows of length 4\*2^n. +//! In particular this means that a 2^n by 2^n matrix contains exactly 2^(2\*n-1) data points when n>0. +//! +//! This structure allows for a multilinear point evaluation of the associated MLE to be represented as a +//! relatively simple matrix product. +//! +//! Concretely, if the data being committed to is 0, 1, 2, 3, etc., the matrix is filled as shown below. +//! +//! ```text +//! 0 +//! _, 1 +//! 2, 3 +//! 4, 5, 6, 7 +//! 8, 9, 10, 11 +//! 12, 13, 14, 15 +//! 16, 17, 18, 19, 20, 21, 22, 23 +//! 24, 25, 26, 27, 28, 29, 30, 31 +//! 32, 33, 34, 35, 36, 37, 38, 39 +//! 40, 41, 42, 43, 44, 45, 46, 47 +//! 48, 49, 50, 51, 52, 53, 54, 55 +//! 56, 57, 58, 59, 60, 61, 62, 63 +//! 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79 +//! 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95 +//! 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111 +//! 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127 +//! ... +//! ``` + +/// Returns the index that belongs in the first column in a particular row. +/// +/// Note: when row = 1, this correctly returns 0, even though no data belongs there. +#[cfg(test)] +pub(crate) fn row_start_index(row: usize) -> usize { + let width_of_row = ((2 * row + 4) / 3).next_power_of_two(); + width_of_row * (row - width_of_row / 2) +} +/// Returns the (row, column) in the matrix where the data with the given index belongs. +#[cfg(test)] +pub(crate) fn row_and_column_from_index(index: usize) -> (usize, usize) { + let width_of_row = 1 << (((2 * index + 1).ilog2() + 1) / 2); + let row = index / width_of_row + width_of_row / 2; + let column = index % width_of_row; + (row, column) +} + +#[cfg(test)] +mod tests { + use super::*; + /// Creates a 2^nu by 2^nu matrix that is partially filled with the indexes that belong in each position. + fn create_position_matrix(nu: usize) -> Vec>> { + let max_index = 1 << (2 * nu - 1); + let mut matrix = vec![]; + for i in 0..max_index { + let (row, column) = row_and_column_from_index(i); + if matrix.len() <= row { + matrix.resize_with(row + 1, Vec::new); + } + if matrix[row].len() <= column { + matrix[row].resize(column + 1, None); + } + matrix[row][column] = Some(i); + } + matrix + } + #[test] + fn we_can_compute_positions_from_indexes() { + assert_eq!( + create_position_matrix(2), + vec![ + vec![Some(0)], // length 1 + vec![None, Some(1)], // length "1" + vec![Some(2), Some(3)], // length 2 + vec![Some(4), Some(5), Some(6), Some(7)], // length 4 + ], + ); + assert_eq!( + create_position_matrix(4), + vec![ + vec![Some(0)], // length 1 + vec![None, Some(1)], // length "1" + vec![Some(2), Some(3)], // length 2 + vec![Some(4), Some(5), Some(6), Some(7)], // length 4 + vec![Some(8), Some(9), Some(10), Some(11)], // length 4 + vec![Some(12), Some(13), Some(14), Some(15)], // length 4 + (16..=23).map(Some).collect(), // length 8 + (24..=31).map(Some).collect(), // length 8 + (32..=39).map(Some).collect(), // length 8 + (40..=47).map(Some).collect(), // length 8 + (48..=55).map(Some).collect(), // length 8 + (56..=63).map(Some).collect(), // length 8 + (64..=79).map(Some).collect(), // length 16 + (80..=95).map(Some).collect(), // length 16 + (96..=111).map(Some).collect(), // length 16 + (112..=127).map(Some).collect() // length 16 + ], + ); + } + #[test] + fn we_can_fill_matrix_with_no_collisions_and_correct_size_and_row_starts() { + for nu in 1..10 { + let num_vars = nu * 2 - 1; + let matrix = create_position_matrix(nu); + // Every position should be unique. + assert_eq!( + matrix.iter().flatten().filter(|&x| x.is_some()).count(), + 1 << num_vars + ); + // The matrix should have 2^nu rows. + assert_eq!(matrix.len(), 1 << nu); + // The matrix should have 2^nu columns. + assert_eq!(matrix.iter().map(Vec::len).max().unwrap(), 1 << nu); + for (index, row) in matrix.iter().enumerate() { + // The start of each row should match with `row_start_index`. + assert_eq!( + row_start_index(index), + match index { + 1 => 0, // Row 1 starts at 0, even though nothing is put in the 0th position. This is because the 0th index is at position (0, 0) + _ => row[0] + .expect("Every row except 1 should have a value in the 0th position."), + } + ) + } + } + } +} diff --git a/crates/proof-of-sql/src/proof_primitive/dory/mod.rs b/crates/proof-of-sql/src/proof_primitive/dory/mod.rs index c49672ff9..317de744f 100644 --- a/crates/proof-of-sql/src/proof_primitive/dory/mod.rs +++ b/crates/proof-of-sql/src/proof_primitive/dory/mod.rs @@ -140,3 +140,5 @@ type DeferredG2 = deferred_msm::DeferredMSM; mod pairings; mod transpose; + +mod dynamic_dory_structure; From cc57de7a8f33dc93748871463accb75b25071d62 Mon Sep 17 00:00:00 2001 From: Jay White Date: Mon, 29 Jul 2024 22:02:16 -0400 Subject: [PATCH 2/2] feat: add dynamic dory basis vector method The `compute_dynamic_standard_basis_vecs` is a helper method used to comput the vectors in the Vector-Matrix-Vector product in the dynamic dory scheme. --- .../dynamic_dory_standard_basis_helper.rs | 281 ++++++++++++++++++ .../src/proof_primitive/dory/mod.rs | 1 + 2 files changed, 282 insertions(+) create mode 100644 crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_standard_basis_helper.rs diff --git a/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_standard_basis_helper.rs b/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_standard_basis_helper.rs new file mode 100644 index 000000000..d38040f9b --- /dev/null +++ b/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_standard_basis_helper.rs @@ -0,0 +1,281 @@ +//! This module provides the `build_standard_basis_vecs` method, which is used in converting a point to a +//! vector used in a Vector-Matrix-Vector product in the dynamic dory scheme. + +use super::F; +use ark_ff::Field; + +#[allow(dead_code)] +/// This method produces evaluation vectors from a point. This is a helper method for generating a Vector-Matrix-Vector product in the dynamic dory scheme. +/// +/// The ith element of the lo_vec is essentially the ith monomial basis element (lexicographically). +/// The ith element of the hi_vec is essentially the jth monomial basis element where j = row_start_index(i). +/// +/// NOTE: the lo_vec and hi_vec are scaled by lo_vec[0] and hi_vec[0] respectively. +/// NOTE: lo_vec and hi_vec should otherwise consist entirely of zeros in order to ensure correct output. +pub(super) fn compute_dynamic_standard_basis_vecs(point: &[F], lo_vec: &mut [F], hi_vec: &mut [F]) { + let nu = point.len() / 2 + 1; + debug_assert_eq!(lo_vec.len(), 1 << nu); + debug_assert_eq!(hi_vec.len(), 1 << nu); + for i in 1..nu { + build_partial_second_half_standard_basis_vecs( + &point[..2 * i - 1], + &mut lo_vec[..1 << i], + &mut hi_vec[..1 << i], + true, + ); + } + // Note: if we don't have the "full" point, we shouldn't fill up the last quarter because it should be all zeros. + build_partial_second_half_standard_basis_vecs(point, lo_vec, hi_vec, point.len() % 2 == 1); + // Add the most significant variable, which was not included before in order to allow simple copying to work. + point.iter().skip(1).enumerate().for_each(|(i, v)| { + let p = i / 2; + let o = 2 + i % 2; + (o << p..(o + 1) << p).for_each(|k| hi_vec[k] *= v) + }); +} + +fn build_partial_second_half_standard_basis_vecs( + point: &[F], + lo_vec: &mut [F], + hi_vec: &mut [F], + add_last_quarter: bool, +) { + let nu = point.len() / 2 + 1; + debug_assert_eq!(lo_vec.len(), 1 << nu); + debug_assert_eq!(hi_vec.len(), 1 << nu); + if nu == 1 { + lo_vec[1] = if point.is_empty() { + F::ZERO + } else { + lo_vec[0] * point[0] + }; + hi_vec[1] = hi_vec[0]; + } else { + let (lo_half0, lo_half1) = lo_vec.split_at_mut(1 << (nu - 1)); + lo_half0 + .iter() + .zip(lo_half1) + .for_each(|(l, h)| *h = *l * point[nu - 1]); + if nu == 2 { + hi_vec[2] = hi_vec[0]; + if add_last_quarter { + hi_vec[3] = hi_vec[1]; + } + } else { + let (hi_half0, hi_half1) = hi_vec.split_at_mut(1 << (nu - 1)); + let (_, hi_quarter1) = hi_half0.split_at(1 << (nu - 2)); + let (hi_quarter2, hi_quarter3) = hi_half1.split_at_mut(1 << (nu - 2)); + let (_, hi_eighth3) = hi_quarter1.split_at(1 << (nu - 3)); + let (hi_eighth4, hi_eighth5) = hi_quarter2.split_at_mut(1 << (nu - 3)); + let (hi_eighth6, hi_eighth7) = hi_quarter3.split_at_mut(1 << (nu - 3)); + // Fill up quarter #2 (from 2/4..3/4). + hi_eighth3 + .iter() + .zip(hi_eighth4.iter_mut().zip(hi_eighth5)) + .for_each(|(&source, (target_lo, target_hi))| { + // Copy eighth #3 (from 3/8..4/8) to eighth #4 (4/8..5/8). + *target_lo = source; + // Copy eighth #3 (from 3/8..4/8) to eighth #5 (5/8..6/8) + // and multiply by the third from the last element in point. + *target_hi = source * point[2 * nu - 4]; + }); + if add_last_quarter { + // Fill up quarter #4 (from 3/4..4/4). + hi_quarter2 + .iter() + .step_by(2) + .zip(hi_eighth6.iter_mut().zip(hi_eighth7)) + .for_each(|(&source, (target_lo, target_hi))| { + // Copy every other in quarter #2 (from 2/4..3/4) to eighth #6 (6/8..7/8). + *target_lo = source; + // Copy every other in quarter #2 (from 2/4..3/4) to eighth #6 (7/8..8/8). + // and multiply by the second from the last element in point. + *target_hi = source * point[2 * nu - 3]; + }); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::{super::dynamic_dory_structure::row_start_index, *}; + + #[test] + fn we_can_compute_dynamic_standard_basis_vecs_from_length_0_point() { + let mut lo_vec = vec![F::ZERO; 2]; + let mut hi_vec = vec![F::ZERO; 2]; + lo_vec[0] = F::from(2); + hi_vec[0] = F::from(3); + let point = vec![]; + let lo_vec_expected = vec![F::from(2), F::ZERO]; + let hi_vec_expected = vec![F::from(3), F::from(3)]; + compute_dynamic_standard_basis_vecs(&point, &mut lo_vec, &mut hi_vec); + assert_eq!(lo_vec, lo_vec_expected); + assert_eq!(hi_vec, hi_vec_expected); + } + #[test] + fn we_can_compute_dynamic_standard_basis_vecs_from_length_1_point() { + let mut lo_vec = vec![F::ZERO; 2]; + let mut hi_vec = vec![F::ZERO; 2]; + lo_vec[0] = F::from(2); + hi_vec[0] = F::from(3); + let point = vec![F::from(5)]; + let lo_vec_expected = vec![F::from(2), F::from(2 * 5)]; + let hi_vec_expected = vec![F::from(3), F::from(3)]; + compute_dynamic_standard_basis_vecs(&point, &mut lo_vec, &mut hi_vec); + assert_eq!(lo_vec, lo_vec_expected); + assert_eq!(hi_vec, hi_vec_expected); + } + #[test] + fn we_can_compute_dynamic_standard_basis_vecs_from_length_2_point() { + let mut lo_vec = vec![F::ZERO; 4]; + let mut hi_vec = vec![F::ZERO; 4]; + lo_vec[0] = F::from(2); + hi_vec[0] = F::from(3); + let point = vec![F::from(5), F::from(7)]; + let lo_vec_expected = vec![ + F::from(2), + F::from(2 * 5), + F::from(2 * 7), + F::from(2 * 5 * 7), + ]; + let hi_vec_expected = vec![F::from(3), F::from(3), F::from(3 * 7), F::ZERO]; + compute_dynamic_standard_basis_vecs(&point, &mut lo_vec, &mut hi_vec); + assert_eq!(lo_vec, lo_vec_expected); + assert_eq!(hi_vec, hi_vec_expected); + } + #[test] + fn we_can_compute_dynamic_standard_basis_vecs_from_length_3_point() { + let mut lo_vec = vec![F::ZERO; 4]; + let mut hi_vec = vec![F::ZERO; 4]; + lo_vec[0] = F::from(2); + hi_vec[0] = F::from(3); + let point = vec![F::from(5), F::from(7), F::from(11)]; + let lo_vec_expected = vec![ + F::from(2), + F::from(2 * 5), + F::from(2 * 7), + F::from(2 * 5 * 7), + ]; + let hi_vec_expected = vec![F::from(3), F::from(3), F::from(3 * 7), F::from(3 * 11)]; + compute_dynamic_standard_basis_vecs(&point, &mut lo_vec, &mut hi_vec); + assert_eq!(lo_vec, lo_vec_expected); + assert_eq!(hi_vec, hi_vec_expected); + } + #[test] + fn we_can_compute_dynamic_standard_basis_vecs_from_length_4_point() { + let mut lo_vec = vec![F::ZERO; 8]; + let mut hi_vec = vec![F::ZERO; 8]; + lo_vec[0] = F::from(2); + hi_vec[0] = F::from(3); + let point = vec![F::from(5), F::from(7), F::from(11), F::from(13)]; + let lo_vec_expected = vec![ + F::from(2), + F::from(2 * 5), + F::from(2 * 7), + F::from(2 * 5 * 7), + F::from(2 * 11), + F::from(2 * 5 * 11), + F::from(2 * 7 * 11), + F::from(2 * 5 * 7 * 11), + ]; + let hi_vec_expected = vec![ + F::from(3), + F::from(3), + F::from(3 * 7), + F::from(3 * 11), + F::from(3 * 13), + F::from(3 * 11 * 13), + F::ZERO, + F::ZERO, + ]; + compute_dynamic_standard_basis_vecs(&point, &mut lo_vec, &mut hi_vec); + assert_eq!(lo_vec, lo_vec_expected); + assert_eq!(hi_vec, hi_vec_expected); + } + #[test] + fn we_can_compute_dynamic_standard_basis_vecs_from_length_5_point() { + let mut lo_vec = vec![F::ZERO; 8]; + let mut hi_vec = vec![F::ZERO; 8]; + lo_vec[0] = F::from(2); + hi_vec[0] = F::from(3); + let point = vec![ + F::from(5), + F::from(7), + F::from(11), + F::from(13), + F::from(17), + ]; + let lo_vec_expected = vec![ + F::from(2), + F::from(2 * 5), + F::from(2 * 7), + F::from(2 * 5 * 7), + F::from(2 * 11), + F::from(2 * 5 * 11), + F::from(2 * 7 * 11), + F::from(2 * 5 * 7 * 11), + ]; + let hi_vec_expected = vec![ + F::from(3), + F::from(3), + F::from(3 * 7), + F::from(3 * 11), + F::from(3 * 13), + F::from(3 * 11 * 13), + F::from(3 * 17), + F::from(3 * 13 * 17), + ]; + compute_dynamic_standard_basis_vecs(&point, &mut lo_vec, &mut hi_vec); + assert_eq!(lo_vec, lo_vec_expected); + assert_eq!(hi_vec, hi_vec_expected); + } + + /// Computes the evaluation of a basis monomial at the given point. + /// + /// In other words, the result is `prod point[i]^(b[i])` where + /// `index = sum 2^i*b[i]` and `b[i]` is `0` or `1`. (i.e. `b` is the binary representation of `index`.) + /// Note: `point` is padded with zeros as needed. + /// + /// This method is primarily to test the `build_standard_basis_vecs` method. + fn get_binary_eval(index: usize, point: &[F]) -> F { + core::iter::successors(Some(index), |&k| match k >> 1 { + 0 => None, + k => Some(k), + }) + .enumerate() + .filter_map(|(i, b)| match b % 2 == 0 { + true => None, + false => Some(point.get(i).copied().unwrap_or(F::ZERO)), + }) + .product() + } + + #[test] + fn we_can_compute_dynamic_random_standard_basis_vecs() { + use ark_std::{test_rng, UniformRand}; + use itertools::Itertools; + let mut rng = test_rng(); + for num_vars in 0..10 { + let point = core::iter::repeat_with(|| F::rand(&mut rng)) + .take(num_vars) + .collect_vec(); + let alpha = F::rand(&mut rng); + let beta = F::rand(&mut rng); + let nu = point.len() / 2 + 1; + let mut lo_vec = vec![F::ZERO; 1 << nu]; + let mut hi_vec = vec![F::ZERO; 1 << nu]; + lo_vec[0] = alpha; + hi_vec[0] = beta; + compute_dynamic_standard_basis_vecs(&point, &mut lo_vec, &mut hi_vec); + for i in 0..1 << nu { + assert_eq!(lo_vec[i], alpha * get_binary_eval(i, &point)); + assert_eq!( + hi_vec[i], + beta * get_binary_eval(row_start_index(i), &point) + ); + } + } + } +} diff --git a/crates/proof-of-sql/src/proof_primitive/dory/mod.rs b/crates/proof-of-sql/src/proof_primitive/dory/mod.rs index 317de744f..ae069a8c2 100644 --- a/crates/proof-of-sql/src/proof_primitive/dory/mod.rs +++ b/crates/proof-of-sql/src/proof_primitive/dory/mod.rs @@ -141,4 +141,5 @@ type DeferredG2 = deferred_msm::DeferredMSM; mod pairings; mod transpose; +mod dynamic_dory_standard_basis_helper; mod dynamic_dory_structure;