From b8637fd5645aaebe0eb888b70cc18ca285690d3a Mon Sep 17 00:00:00 2001 From: Jacob Trombetta Date: Mon, 21 Oct 2024 13:47:52 -0400 Subject: [PATCH 1/4] perf: dynamic dory commitment computation should be able to be done in parallel --- .../dory/blitzar_metadata_table.rs | 167 ++++++++++++------ .../dynamic_dory_commitment_helper_gpu.rs | 43 +++-- 2 files changed, 142 insertions(+), 68 deletions(-) diff --git a/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs b/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs index 72e2ca8cb..f85d96dc7 100644 --- a/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs +++ b/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs @@ -5,7 +5,7 @@ use super::{ G1Affine, F, }; use crate::{ - base::{commitment::CommittableColumn, database::ColumnType}, + base::{commitment::CommittableColumn, database::ColumnType, if_rayon}, proof_primitive::dory::offset_to_bytes::OffsetToBytes, }; use alloc::{vec, vec::Vec}; @@ -14,6 +14,11 @@ use ark_ff::MontFp; use ark_std::ops::Mul; use core::iter; use itertools::Itertools; +#[cfg(feature = "rayon")] +use rayon::{ + iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator}, + prelude::ParallelSliceMut, +}; use tracing::{span, Level}; const BYTE_SIZE: u32 = 8; @@ -72,11 +77,18 @@ pub fn signed_commits( } } - unsigned_sub_commits - .into_iter() - .zip(min_sub_commits.into_iter()) - .map(|(unsigned, min)| (unsigned + min).into()) - .collect() + if_rayon!( + unsigned_sub_commits + .into_par_iter() + .zip(min_sub_commits.into_par_iter()) + .map(|(signed, offset)| (signed + offset).into()) + .collect(), + unsigned_sub_commits + .into_iter() + .zip(min_sub_commits.into_iter()) + .map(|(unsigned, min)| (unsigned + min).into()) + .collect() + ) } /// Copies the column data to the scalar row slice. @@ -123,7 +135,61 @@ fn copy_column_data_to_slice( } } -#[allow(clippy::cast_possible_truncation)] +/// Populates a slice of the scalar array with committable column data +/// while adding the ones entry to handle signed data columns. +/// +/// # Arguments +/// +/// * `scalar_col` - The scalar column index. +/// * `scalar_row` - The scalar row index. +/// * `scalar_row_slice` - A mutable reference to the scalar row slice. +/// * `offset` - The offset to the data. +/// * `committable_columns` - A reference to the committable columns. +/// * `cumulative_byte_length_table` - A reference to the cumulative byte length table. +/// * `single_entry_in_blitzar_output_bit_table` - A reference to the single entry in the Blitzar output bit table. +/// * `ones_columns_lengths` - A reference to the ones columns lengths. +/// * `num_of_bytes_in_committable_columns` - The number of bytes in the committable columns. +#[allow(clippy::too_many_arguments)] +fn populate_scalar_array_with_column( + scalar_col: usize, + scalar_row: usize, + scalar_row_slice: &mut [u8], + offset: usize, + committable_columns: &[CommittableColumn], + cumulative_byte_length_table: &[usize], + single_entry_in_blitzar_output_bit_table: &[u32], + ones_columns_lengths: &[usize], + num_of_bytes_in_committable_columns: usize, +) { + if let Some(index) = index_from_row_and_column(scalar_col, scalar_row) + .and_then(|committable_column_idx| committable_column_idx.checked_sub(offset)) + { + for (i, committable_column) in committable_columns + .iter() + .enumerate() + .filter(|(_, committable_column)| index < committable_column.len()) + { + let start = cumulative_byte_length_table + [i + scalar_col * single_entry_in_blitzar_output_bit_table.len()]; + let end = start + (single_entry_in_blitzar_output_bit_table[i] / BYTE_SIZE) as usize; + + copy_column_data_to_slice(committable_column, scalar_row_slice, start, end, index); + } + + ones_columns_lengths + .iter() + .positions(|ones_columns_length| index < *ones_columns_length) + .for_each(|i| { + let ones_index = i + + scalar_col + * (num_of_bytes_in_committable_columns + ones_columns_lengths.len()) + + num_of_bytes_in_committable_columns; + + scalar_row_slice[ones_index] = 1_u8; + }); + } +} + /// Creates the metadata tables for Blitzar's `vlen_msm` algorithm. /// /// # Arguments @@ -185,10 +251,11 @@ pub fn create_blitzar_metadata_tables( / single_entry_in_blitzar_output_bit_table.len()) .flat_map(|i| { itertools::repeat_n( - full_width_of_row(i + offset_row) as u32, + u32::try_from(full_width_of_row(i)), single_entry_in_blitzar_output_bit_table.len(), ) }) + .flatten() .collect(); // Create a cumulative length table to be used when packing the scalar vector. @@ -208,54 +275,44 @@ pub fn create_blitzar_metadata_tables( // Populate the scalars array. let span = span!(Level::INFO, "pack_blitzar_scalars").entered(); if !blitzar_scalars.is_empty() { - blitzar_scalars - .chunks_exact_mut(num_scalar_columns) - .enumerate() - .for_each(|(scalar_row, scalar_row_slice)| { - // Iterate over the columns and populate the scalars array. - for scalar_col in 0..offset_height { - // Find index in the committable columns. Note, the scalar is in - // column major order, that is why the (row, col) arguments are flipped. - if let Some(index) = - index_from_row_and_column(scalar_col + offset_row, scalar_row).and_then( - |committable_column_idx| committable_column_idx.checked_sub(offset), - ) - { - for (i, committable_column) in committable_columns - .iter() - .enumerate() - .filter(|(_, committable_column)| index < committable_column.len()) - { - let start = cumulative_byte_length_table - [i + scalar_col * single_entry_in_blitzar_output_bit_table.len()]; - let end = start - + (single_entry_in_blitzar_output_bit_table[i] / BYTE_SIZE) - as usize; - - copy_column_data_to_slice( - committable_column, - scalar_row_slice, - start, - end, - index, - ); - } - - ones_columns_lengths - .iter() - .positions(|ones_columns_length| index < *ones_columns_length) - .for_each(|i| { - let ones_index = i - + scalar_col - * (num_of_bytes_in_committable_columns - + ones_columns_lengths.len()) - + num_of_bytes_in_committable_columns; - - scalar_row_slice[ones_index] = 1_u8; - }); + if_rayon!( + blitzar_scalars + .par_chunks_exact_mut(num_scalar_columns) + .enumerate() + .for_each(|(scalar_row, scalar_row_slice)| { + for scalar_col in 0..max_height { + populate_scalar_array_with_column( + scalar_col, + scalar_row, + scalar_row_slice, + offset, + committable_columns, + &cumulative_byte_length_table, + &single_entry_in_blitzar_output_bit_table, + &ones_columns_lengths, + num_of_bytes_in_committable_columns, + ); } - } - }); + }), + blitzar_scalars + .chunks_exact_mut(num_scalar_columns) + .enumerate() + .for_each(|(scalar_row, scalar_row_slice)| { + for scalar_col in 0..max_height { + populate_scalar_array_with_column( + scalar_col, + scalar_row, + scalar_row_slice, + offset, + committable_columns, + &cumulative_byte_length_table, + &single_entry_in_blitzar_output_bit_table, + &ones_columns_lengths, + num_of_bytes_in_committable_columns, + ); + } + }) + ); } span.exit(); diff --git a/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_commitment_helper_gpu.rs b/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_commitment_helper_gpu.rs index 7a5c8c022..8327ca598 100644 --- a/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_commitment_helper_gpu.rs +++ b/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_commitment_helper_gpu.rs @@ -3,8 +3,10 @@ use super::{ dynamic_dory_structure::row_and_column_from_index, pairings, DynamicDoryCommitment, G1Affine, ProverSetup, }; -use crate::base::{commitment::CommittableColumn, slice_ops::slice_cast}; +use crate::base::{commitment::CommittableColumn, if_rayon, slice_ops::slice_cast}; use blitzar::compute::ElementP2; +#[cfg(feature = "rayon")] +use rayon::iter::{IntoParallelIterator, ParallelIterator}; use tracing::{span, Level}; /// Computes the dynamic Dory commitment using the GPU implementation of the `vlen_msm` algorithm. @@ -69,18 +71,33 @@ pub(super) fn compute_dynamic_dory_commitments( committable_columns.len() ]) .unwrap_or_else(|| { - (0..committable_columns.len()) - .map(|i| { - let sub_slice = signed_sub_commits[i..] - .iter() - .step_by(committable_columns.len()) - .take(num_commits); - DynamicDoryCommitment(pairings::multi_pairing( - sub_slice, - &Gamma_2[gamma_2_offset..gamma_2_offset + num_commits], - )) - }) - .collect() + if_rayon!( + (0..committable_columns.len()) + .into_par_iter() + .map(|i| { + let sub_slice = signed_sub_commits[i..] + .iter() + .step_by(committable_columns.len()) + .take(num_commits); + DynamicDoryCommitment(pairings::multi_pairing( + sub_slice, + &Gamma_2[..num_commits], + )) + }) + .collect(), + (0..committable_columns.len()) + .map(|i| { + let sub_slice = signed_sub_commits[i..] + .iter() + .step_by(committable_columns.len()) + .take(num_commits); + DynamicDoryCommitment(pairings::multi_pairing( + sub_slice, + &Gamma_2[..num_commits], + )) + }) + .collect() + ) }); span.exit(); From 65a246d029c847dcfb2bfc51e4f3324ca6e2479b Mon Sep 17 00:00:00 2001 From: Jacob Trombetta Date: Mon, 21 Oct 2024 16:46:14 -0400 Subject: [PATCH 2/4] refactor: improve usage of if_rayon macro in dynamic_dory_commitment_helper_gpu and blitzar_metadata_table modules --- .../dory/blitzar_metadata_table.rs | 139 ++++++------------ .../dynamic_dory_commitment_helper_gpu.rs | 33 ++--- 2 files changed, 54 insertions(+), 118 deletions(-) diff --git a/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs b/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs index f85d96dc7..ea81d9a69 100644 --- a/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs +++ b/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs @@ -80,15 +80,13 @@ pub fn signed_commits( if_rayon!( unsigned_sub_commits .into_par_iter() - .zip(min_sub_commits.into_par_iter()) - .map(|(signed, offset)| (signed + offset).into()) - .collect(), + .zip(min_sub_commits.into_par_iter()), unsigned_sub_commits .into_iter() .zip(min_sub_commits.into_iter()) - .map(|(unsigned, min)| (unsigned + min).into()) - .collect() ) + .map(|(unsigned, min)| (unsigned + min).into()) + .collect() } /// Copies the column data to the scalar row slice. @@ -135,61 +133,6 @@ fn copy_column_data_to_slice( } } -/// Populates a slice of the scalar array with committable column data -/// while adding the ones entry to handle signed data columns. -/// -/// # Arguments -/// -/// * `scalar_col` - The scalar column index. -/// * `scalar_row` - The scalar row index. -/// * `scalar_row_slice` - A mutable reference to the scalar row slice. -/// * `offset` - The offset to the data. -/// * `committable_columns` - A reference to the committable columns. -/// * `cumulative_byte_length_table` - A reference to the cumulative byte length table. -/// * `single_entry_in_blitzar_output_bit_table` - A reference to the single entry in the Blitzar output bit table. -/// * `ones_columns_lengths` - A reference to the ones columns lengths. -/// * `num_of_bytes_in_committable_columns` - The number of bytes in the committable columns. -#[allow(clippy::too_many_arguments)] -fn populate_scalar_array_with_column( - scalar_col: usize, - scalar_row: usize, - scalar_row_slice: &mut [u8], - offset: usize, - committable_columns: &[CommittableColumn], - cumulative_byte_length_table: &[usize], - single_entry_in_blitzar_output_bit_table: &[u32], - ones_columns_lengths: &[usize], - num_of_bytes_in_committable_columns: usize, -) { - if let Some(index) = index_from_row_and_column(scalar_col, scalar_row) - .and_then(|committable_column_idx| committable_column_idx.checked_sub(offset)) - { - for (i, committable_column) in committable_columns - .iter() - .enumerate() - .filter(|(_, committable_column)| index < committable_column.len()) - { - let start = cumulative_byte_length_table - [i + scalar_col * single_entry_in_blitzar_output_bit_table.len()]; - let end = start + (single_entry_in_blitzar_output_bit_table[i] / BYTE_SIZE) as usize; - - copy_column_data_to_slice(committable_column, scalar_row_slice, start, end, index); - } - - ones_columns_lengths - .iter() - .positions(|ones_columns_length| index < *ones_columns_length) - .for_each(|i| { - let ones_index = i - + scalar_col - * (num_of_bytes_in_committable_columns + ones_columns_lengths.len()) - + num_of_bytes_in_committable_columns; - - scalar_row_slice[ones_index] = 1_u8; - }); - } -} - /// Creates the metadata tables for Blitzar's `vlen_msm` algorithm. /// /// # Arguments @@ -276,43 +219,51 @@ pub fn create_blitzar_metadata_tables( let span = span!(Level::INFO, "pack_blitzar_scalars").entered(); if !blitzar_scalars.is_empty() { if_rayon!( - blitzar_scalars - .par_chunks_exact_mut(num_scalar_columns) - .enumerate() - .for_each(|(scalar_row, scalar_row_slice)| { - for scalar_col in 0..max_height { - populate_scalar_array_with_column( - scalar_col, - scalar_row, + blitzar_scalars.par_chunks_exact_mut(num_scalar_columns), + blitzar_scalars.chunks_exact_mut(num_scalar_columns) + ) + .enumerate() + .for_each(|(scalar_row, scalar_row_slice)| { + for scalar_col in 0..max_height { + // Find index in the committable columns. Note, the scalar is in + // column major order, that is why the (row, col) arguments are flipped. + if let Some(index) = index_from_row_and_column(scalar_col, scalar_row) + .and_then(|committable_column_idx| committable_column_idx.checked_sub(offset)) + { + for (i, committable_column) in committable_columns + .iter() + .enumerate() + .filter(|(_, committable_column)| index < committable_column.len()) + { + let start = cumulative_byte_length_table + [i + scalar_col * single_entry_in_blitzar_output_bit_table.len()]; + let end = start + + (single_entry_in_blitzar_output_bit_table[i] / BYTE_SIZE) as usize; + + copy_column_data_to_slice( + committable_column, scalar_row_slice, - offset, - committable_columns, - &cumulative_byte_length_table, - &single_entry_in_blitzar_output_bit_table, - &ones_columns_lengths, - num_of_bytes_in_committable_columns, + start, + end, + index, ); } - }), - blitzar_scalars - .chunks_exact_mut(num_scalar_columns) - .enumerate() - .for_each(|(scalar_row, scalar_row_slice)| { - for scalar_col in 0..max_height { - populate_scalar_array_with_column( - scalar_col, - scalar_row, - scalar_row_slice, - offset, - committable_columns, - &cumulative_byte_length_table, - &single_entry_in_blitzar_output_bit_table, - &ones_columns_lengths, - num_of_bytes_in_committable_columns, - ); - } - }) - ); + + ones_columns_lengths + .iter() + .positions(|ones_columns_length| index < *ones_columns_length) + .for_each(|i| { + let ones_index = i + + scalar_col + * (num_of_bytes_in_committable_columns + + ones_columns_lengths.len()) + + num_of_bytes_in_committable_columns; + + scalar_row_slice[ones_index] = 1_u8; + }); + } + } + }); } span.exit(); diff --git a/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_commitment_helper_gpu.rs b/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_commitment_helper_gpu.rs index 8327ca598..7687033df 100644 --- a/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_commitment_helper_gpu.rs +++ b/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_commitment_helper_gpu.rs @@ -72,32 +72,17 @@ pub(super) fn compute_dynamic_dory_commitments( ]) .unwrap_or_else(|| { if_rayon!( + (0..committable_columns.len()).into_par_iter(), (0..committable_columns.len()) - .into_par_iter() - .map(|i| { - let sub_slice = signed_sub_commits[i..] - .iter() - .step_by(committable_columns.len()) - .take(num_commits); - DynamicDoryCommitment(pairings::multi_pairing( - sub_slice, - &Gamma_2[..num_commits], - )) - }) - .collect(), - (0..committable_columns.len()) - .map(|i| { - let sub_slice = signed_sub_commits[i..] - .iter() - .step_by(committable_columns.len()) - .take(num_commits); - DynamicDoryCommitment(pairings::multi_pairing( - sub_slice, - &Gamma_2[..num_commits], - )) - }) - .collect() ) + .map(|i| { + let sub_slice = signed_sub_commits[i..] + .iter() + .step_by(committable_columns.len()) + .take(num_commits); + DynamicDoryCommitment(pairings::multi_pairing(sub_slice, &Gamma_2[..num_commits])) + }) + .collect() }); span.exit(); From cf406ed74f0d8a6108ba80eb9ba5f626972c72e1 Mon Sep 17 00:00:00 2001 From: Jacob Trombetta Date: Mon, 21 Oct 2024 20:14:19 -0400 Subject: [PATCH 3/4] refactor: add panic if the row index returned exceeds u32::MAX in the blitzar_metadata_table module --- .../src/proof_primitive/dory/blitzar_metadata_table.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs b/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs index ea81d9a69..7282d1914 100644 --- a/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs +++ b/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs @@ -144,6 +144,10 @@ fn copy_column_data_to_slice( /// /// A tuple containing the output bit table, output length table, /// and scalars required to call Blitzar's `vlen_msm` function. +/// +/// # Panics +/// +/// Panics if the row of a column exceeds `u32::MAX`. #[tracing::instrument(name = "create_blitzar_metadata_tables", level = "debug", skip_all)] pub fn create_blitzar_metadata_tables( committable_columns: &[CommittableColumn], @@ -194,11 +198,11 @@ pub fn create_blitzar_metadata_tables( / single_entry_in_blitzar_output_bit_table.len()) .flat_map(|i| { itertools::repeat_n( - u32::try_from(full_width_of_row(i)), + u32::try_from(full_width_of_row(i)) + .expect("row lengths should never exceed u32::MAX"), single_entry_in_blitzar_output_bit_table.len(), ) }) - .flatten() .collect(); // Create a cumulative length table to be used when packing the scalar vector. From 025c55aa9344db544f58105f8ab86a36b615eefd Mon Sep 17 00:00:00 2001 From: Jacob Trombetta Date: Tue, 22 Oct 2024 10:10:45 -0400 Subject: [PATCH 4/4] fix: fix rebase conflict in dynamic_dory_commitment_helper_gpu and blitzar_metadata_table modules --- .../src/proof_primitive/dory/blitzar_metadata_table.rs | 6 +++--- .../dory/dynamic_dory_commitment_helper_gpu.rs | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs b/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs index 7282d1914..4c24b8d15 100644 --- a/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs +++ b/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs @@ -198,7 +198,7 @@ pub fn create_blitzar_metadata_tables( / single_entry_in_blitzar_output_bit_table.len()) .flat_map(|i| { itertools::repeat_n( - u32::try_from(full_width_of_row(i)) + u32::try_from(full_width_of_row(i + offset_row)) .expect("row lengths should never exceed u32::MAX"), single_entry_in_blitzar_output_bit_table.len(), ) @@ -228,10 +228,10 @@ pub fn create_blitzar_metadata_tables( ) .enumerate() .for_each(|(scalar_row, scalar_row_slice)| { - for scalar_col in 0..max_height { + for scalar_col in 0..offset_height { // Find index in the committable columns. Note, the scalar is in // column major order, that is why the (row, col) arguments are flipped. - if let Some(index) = index_from_row_and_column(scalar_col, scalar_row) + if let Some(index) = index_from_row_and_column(scalar_col + offset_row, scalar_row) .and_then(|committable_column_idx| committable_column_idx.checked_sub(offset)) { for (i, committable_column) in committable_columns diff --git a/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_commitment_helper_gpu.rs b/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_commitment_helper_gpu.rs index 7687033df..f3206f7c6 100644 --- a/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_commitment_helper_gpu.rs +++ b/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_commitment_helper_gpu.rs @@ -80,7 +80,10 @@ pub(super) fn compute_dynamic_dory_commitments( .iter() .step_by(committable_columns.len()) .take(num_commits); - DynamicDoryCommitment(pairings::multi_pairing(sub_slice, &Gamma_2[..num_commits])) + DynamicDoryCommitment(pairings::multi_pairing( + sub_slice, + &Gamma_2[gamma_2_offset..gamma_2_offset + num_commits], + )) }) .collect() });