From f98b2bb60f7ca52aa89ae0c01160200dbe606608 Mon Sep 17 00:00:00 2001 From: Jacob Trombetta Date: Mon, 21 Oct 2024 16:14:58 -0400 Subject: [PATCH 1/6] perf: dynamic Dory commitment computation should efficiently handle the offset to avoid unnecessary computation --- .../dory/blitzar_metadata_table.rs | 201 +++++++++++++++--- .../dynamic_dory_commitment_helper_gpu.rs | 4 +- 2 files changed, 179 insertions(+), 26 deletions(-) diff --git a/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs b/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs index c4a0138cc..a9f3ff789 100644 --- a/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs +++ b/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs @@ -1,5 +1,7 @@ use super::{ - dynamic_dory_structure::{full_width_of_row, index_from_row_and_column, matrix_size}, + dynamic_dory_structure::{ + full_width_of_row, index_from_row_and_column, matrix_size, row_and_column_from_index, + }, G1Affine, F, }; use crate::{ @@ -153,6 +155,10 @@ pub fn create_blitzar_metadata_tables( (0, 0) }; + // We will ignore the rows that are zero from the offsets. + let offset_row = row_and_column_from_index(offset).0; + let offset_height = max_height - offset_row; + // Find the single packed byte size of all committable columns. let num_of_bytes_in_committable_columns: usize = committable_columns .iter() @@ -171,7 +177,7 @@ pub fn create_blitzar_metadata_tables( .iter() .copied() .cycle() - .take(single_entry_in_blitzar_output_bit_table.len() * max_height) + .take(single_entry_in_blitzar_output_bit_table.len() * offset_height) .collect(); // Create the full length vector to be used by Blitzar's vlen_msm algorithm. @@ -179,7 +185,7 @@ pub fn create_blitzar_metadata_tables( / single_entry_in_blitzar_output_bit_table.len()) .flat_map(|i| { itertools::repeat_n( - full_width_of_row(i) as u32, + full_width_of_row(i + offset_row) as u32, single_entry_in_blitzar_output_bit_table.len(), ) }) @@ -196,7 +202,7 @@ pub fn create_blitzar_metadata_tables( // Create scalars array. Note, scalars need to be stored in a column-major order. let num_scalar_rows = max_width; let num_scalar_columns = - (num_of_bytes_in_committable_columns + ones_columns_lengths.len()) * max_height; + (num_of_bytes_in_committable_columns + ones_columns_lengths.len()) * offset_height; let mut blitzar_scalars = vec![0u8; num_scalar_rows * num_scalar_columns]; // Populate the scalars array. @@ -207,12 +213,14 @@ pub fn create_blitzar_metadata_tables( .enumerate() .for_each(|(scalar_row, scalar_row_slice)| { // Iterate over the columns and populate the scalars array. - for scalar_col in 0..max_height { + for scalar_col in 0..offset_height { // Find index in the committable columns. Note, the scalar is in // column major order, that is why the (row, col) arguments are flipped. - if let Some(index) = index_from_row_and_column(scalar_col, scalar_row).and_then( - |committable_column_idx| committable_column_idx.checked_sub(offset), - ) { + if let Some(index) = + index_from_row_and_column(scalar_col + offset_row, scalar_row).and_then( + |committable_column_idx| committable_column_idx.checked_sub(offset), + ) + { for (i, committable_column) in committable_columns .iter() .enumerate() @@ -265,27 +273,83 @@ mod tests { use proof_of_sql_parser::posql_time::{PoSQLTimeUnit, PoSQLTimeZone}; #[test] - fn we_can_populate_blitzar_metadata_tables_with_empty_columns() { + fn we_can_populate_blitzar_metadata_tables_with_empty_columns_with_full_column_offsets() { let committable_columns = [CommittableColumn::BigInt(&[0; 0])]; - let offset = 0; - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); + let offsets = vec![ + 0, 1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, + ]; + for &offset in &offsets { + let (bit_table, length_table, scalars) = + create_blitzar_metadata_tables(&committable_columns, offset); - assert!(bit_table.is_empty()); - assert!(length_table.is_empty()); - assert!(scalars.is_empty()); + assert!(bit_table.is_empty()); + assert!(length_table.is_empty()); + assert!(scalars.is_empty()); + } } #[test] - fn we_can_populate_blitzar_metadata_tables_with_empty_columns_and_an_offset() { + fn we_can_populate_blitzar_metadata_tables_with_empty_columns_and_an_offset_with_partial_column_offsets( + ) { let committable_columns = [CommittableColumn::BigInt(&[0; 0])]; - let offset = 1; + + let offset = 3; let (bit_table, length_table, scalars) = create_blitzar_metadata_tables(&committable_columns, offset); assert_eq!(bit_table, vec![64, 8]); - assert_eq!(length_table, vec![1, 1]); - assert_eq!(scalars, vec![0, 0, 0, 0, 0, 0, 0, 0, 0]); + assert_eq!(length_table, vec![2, 2]); + assert_eq!( + scalars, + vec![0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + ); + + let offset = 5; + let (bit_table, length_table, scalars) = + create_blitzar_metadata_tables(&committable_columns, offset); + + assert_eq!(bit_table, vec![64, 8]); + assert_eq!(length_table, vec![4, 4]); + assert_eq!( + scalars, + vec![ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 + ] + ); + + let offset = 17; + let (bit_table, length_table, scalars) = + create_blitzar_metadata_tables(&committable_columns, offset); + + assert_eq!(bit_table, vec![64, 8]); + assert_eq!(length_table, vec![8, 8]); + assert_eq!( + scalars, + vec![ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + ] + ); + + let offset = 65; + let (bit_table, length_table, scalars) = + create_blitzar_metadata_tables(&committable_columns, offset); + + assert_eq!(bit_table, vec![64, 8]); + assert_eq!(length_table, vec![16, 16]); + assert_eq!( + scalars, + vec![ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0 + ] + ); } #[test] @@ -307,14 +371,11 @@ mod tests { let (bit_table, length_table, scalars) = create_blitzar_metadata_tables(&committable_columns, offset); - assert_eq!(bit_table, vec![64, 8, 64, 8]); - assert_eq!(length_table, vec![1, 1, 2, 2]); + assert_eq!(bit_table, vec![64, 8]); + assert_eq!(length_table, vec![2, 2]); assert_eq!( scalars, - vec![ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 128, 1 - ] + vec![0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 128, 1] ); } @@ -357,4 +418,94 @@ mod tests { ] ); } + + #[test] + fn we_can_populate_blitzar_metadata_tables_with_mixed_columns_and_partial_column_offset() { + let committable_columns = [ + CommittableColumn::TinyInt(&[1]), + CommittableColumn::SmallInt(&[2]), + CommittableColumn::Int(&[3]), + CommittableColumn::BigInt(&[4]), + CommittableColumn::Int128(&[5]), + CommittableColumn::Decimal75(Precision::new(1).unwrap(), 0, vec![[6, 0, 0, 0]]), + CommittableColumn::Scalar(vec![[7, 0, 0, 0]]), + CommittableColumn::VarChar(vec![[8, 0, 0, 0]]), + CommittableColumn::TimestampTZ(PoSQLTimeUnit::Second, PoSQLTimeZone::Utc, &[9]), + CommittableColumn::Boolean(&[true]), + ]; + + let offset = 1; + let (bit_table, length_table, scalars) = + create_blitzar_metadata_tables(&committable_columns, offset); + assert_eq!( + bit_table, + vec![8, 16, 32, 64, 128, 256, 256, 256, 64, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8] + ); + + assert_eq!( + length_table, + vec![2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2] + ); + assert_eq!( + scalars, + vec![ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 129, 2, 128, 3, 0, 0, 128, 4, 0, 0, 0, 0, 0, 0, 128, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, + 0, 0, 0, 0, 128, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + ] + ); + } + + #[test] + fn we_can_populate_blitzar_metadata_tables_with_mixed_columns_and_full_column_offset() { + let committable_columns = [ + CommittableColumn::TinyInt(&[1]), + CommittableColumn::SmallInt(&[2]), + CommittableColumn::Int(&[3]), + CommittableColumn::BigInt(&[4]), + CommittableColumn::Int128(&[5]), + CommittableColumn::Decimal75(Precision::new(1).unwrap(), 0, vec![[6, 0, 0, 0]]), + CommittableColumn::Scalar(vec![[7, 0, 0, 0]]), + CommittableColumn::VarChar(vec![[8, 0, 0, 0]]), + CommittableColumn::TimestampTZ(PoSQLTimeUnit::Second, PoSQLTimeZone::Utc, &[9]), + CommittableColumn::Boolean(&[true]), + ]; + + let offset = 2; + let (bit_table, length_table, scalars) = + create_blitzar_metadata_tables(&committable_columns, offset); + assert_eq!( + bit_table, + vec![8, 16, 32, 64, 128, 256, 256, 256, 64, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8] + ); + + assert_eq!( + length_table, + vec![2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2] + ); + assert_eq!( + scalars, + vec![ + 129, 2, 128, 3, 0, 0, 128, 4, 0, 0, 0, 0, 0, 0, 128, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 128, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 128, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + ] + ); + } } diff --git a/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_commitment_helper_gpu.rs b/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_commitment_helper_gpu.rs index 4a1f10b86..5759310bc 100644 --- a/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_commitment_helper_gpu.rs +++ b/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_commitment_helper_gpu.rs @@ -1,5 +1,6 @@ use super::{ blitzar_metadata_table::{create_blitzar_metadata_tables, signed_commits}, + dynamic_dory_structure::row_and_column_from_index, pairings, DynamicDoryCommitment, G1Affine, ProverSetup, }; use crate::base::{commitment::CommittableColumn, slice_ops::slice_cast}; @@ -32,6 +33,7 @@ pub(super) fn compute_dynamic_dory_commitments( setup: &ProverSetup, ) -> Vec { let Gamma_2 = setup.Gamma_2.last().unwrap(); + let gamma_2_offset = row_and_column_from_index(offset).0; // Get metadata tables for Blitzar's vlen_msm algorithm. let (blitzar_output_bit_table, blitzar_output_length_table, blitzar_scalars) = @@ -75,7 +77,7 @@ pub(super) fn compute_dynamic_dory_commitments( .take(num_commits); DynamicDoryCommitment(pairings::multi_pairing( sub_slice, - &Gamma_2[..num_commits], + &Gamma_2[gamma_2_offset..gamma_2_offset + num_commits], )) }) .collect() From 35d23b506c299b9b0d6717f639e7b60cec5964e9 Mon Sep 17 00:00:00 2001 From: Jacob Trombetta Date: Mon, 21 Oct 2024 17:02:12 -0400 Subject: [PATCH 2/6] test: add non-trivial offset test with data to blitzar_metadata_table module --- .../dory/blitzar_metadata_table.rs | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs b/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs index a9f3ff789..1b4584e0c 100644 --- a/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs +++ b/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs @@ -379,6 +379,114 @@ mod tests { ); } + #[test] + fn we_can_populate_blitzar_metadata_tables_with_simple_column_and_non_trivial_offsets() { + let committable_columns = [CommittableColumn::TinyInt(&[1])]; + + let offset = 0; + let (bit_table, length_table, scalars) = + create_blitzar_metadata_tables(&committable_columns, offset); + + assert_eq!(bit_table, vec![8, 8]); + assert_eq!(length_table, vec![1, 1]); + assert_eq!(scalars, vec![129, 1]); + + let offset = 1; + let (bit_table, length_table, scalars) = + create_blitzar_metadata_tables(&committable_columns, offset); + + assert_eq!(bit_table, vec![8, 8]); + assert_eq!(length_table, vec![2, 2]); + assert_eq!(scalars, vec![0, 0, 129, 1]); + + let offset = 2; + let (bit_table, length_table, scalars) = + create_blitzar_metadata_tables(&committable_columns, offset); + + assert_eq!(bit_table, vec![8, 8]); + assert_eq!(length_table, vec![2, 2]); + assert_eq!(scalars, vec![129, 1, 0, 0]); + + let offset = 3; + let (bit_table, length_table, scalars) = + create_blitzar_metadata_tables(&committable_columns, offset); + + assert_eq!(bit_table, vec![8, 8]); + assert_eq!(length_table, vec![2, 2]); + assert_eq!(scalars, vec![0, 0, 129, 1]); + + let offset = 4; + let (bit_table, length_table, scalars) = + create_blitzar_metadata_tables(&committable_columns, offset); + + assert_eq!(bit_table, vec![8, 8]); + assert_eq!(length_table, vec![4, 4]); + assert_eq!(scalars, vec![129, 1, 0, 0, 0, 0, 0, 0]); + + let offset = 5; + let (bit_table, length_table, scalars) = + create_blitzar_metadata_tables(&committable_columns, offset); + + assert_eq!(bit_table, vec![8, 8]); + assert_eq!(length_table, vec![4, 4]); + assert_eq!(scalars, vec![0, 0, 129, 1, 0, 0, 0, 0]); + + let offset = 10; + let (bit_table, length_table, scalars) = + create_blitzar_metadata_tables(&committable_columns, offset); + + assert_eq!(bit_table, vec![8, 8]); + assert_eq!(length_table, vec![4, 4]); + assert_eq!(scalars, vec![0, 0, 0, 0, 129, 1, 0, 0]); + + let offset = 16; + let (bit_table, length_table, scalars) = + create_blitzar_metadata_tables(&committable_columns, offset); + + assert_eq!(bit_table, vec![8, 8]); + assert_eq!(length_table, vec![8, 8]); + assert_eq!( + scalars, + vec![129, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + ); + + let offset = 20; + let (bit_table, length_table, scalars) = + create_blitzar_metadata_tables(&committable_columns, offset); + + assert_eq!(bit_table, vec![8, 8]); + assert_eq!(length_table, vec![8, 8]); + assert_eq!( + scalars, + vec![0, 0, 0, 0, 0, 0, 0, 0, 129, 1, 0, 0, 0, 0, 0, 0] + ); + + let offset = 50; + let (bit_table, length_table, scalars) = + create_blitzar_metadata_tables(&committable_columns, offset); + + assert_eq!(bit_table, vec![8, 8]); + assert_eq!(length_table, vec![8, 8]); + assert_eq!( + scalars, + vec![0, 0, 0, 0, 129, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + ); + + let offset = 64; + let (bit_table, length_table, scalars) = + create_blitzar_metadata_tables(&committable_columns, offset); + + assert_eq!(bit_table, vec![8, 8]); + assert_eq!(length_table, vec![16, 16]); + assert_eq!( + scalars, + vec![ + 129, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0 + ] + ); + } + #[test] fn we_can_populate_blitzar_metadata_tables_with_mixed_columns() { let committable_columns = [ From 453445988639077199fd6121a36f92650efa0e42 Mon Sep 17 00:00:00 2001 From: Jacob Trombetta Date: Mon, 21 Oct 2024 17:07:00 -0400 Subject: [PATCH 3/6] chore: remove tuple indexing for readability --- .../src/proof_primitive/dory/blitzar_metadata_table.rs | 2 +- .../proof_primitive/dory/dynamic_dory_commitment_helper_gpu.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs b/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs index 1b4584e0c..30be83197 100644 --- a/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs +++ b/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs @@ -156,7 +156,7 @@ pub fn create_blitzar_metadata_tables( }; // We will ignore the rows that are zero from the offsets. - let offset_row = row_and_column_from_index(offset).0; + let (offset_row, _) = row_and_column_from_index(offset); let offset_height = max_height - offset_row; // Find the single packed byte size of all committable columns. diff --git a/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_commitment_helper_gpu.rs b/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_commitment_helper_gpu.rs index 5759310bc..7a5c8c022 100644 --- a/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_commitment_helper_gpu.rs +++ b/crates/proof-of-sql/src/proof_primitive/dory/dynamic_dory_commitment_helper_gpu.rs @@ -33,7 +33,7 @@ pub(super) fn compute_dynamic_dory_commitments( setup: &ProverSetup, ) -> Vec { let Gamma_2 = setup.Gamma_2.last().unwrap(); - let gamma_2_offset = row_and_column_from_index(offset).0; + let (gamma_2_offset, _) = row_and_column_from_index(offset); // Get metadata tables for Blitzar's vlen_msm algorithm. let (blitzar_output_bit_table, blitzar_output_length_table, blitzar_scalars) = From 540a522fc57dc2183ac184468a221383feda059a Mon Sep 17 00:00:00 2001 From: Jacob Trombetta Date: Mon, 21 Oct 2024 21:10:07 -0400 Subject: [PATCH 4/6] test: add more exhaustive tests for the dory offset calculation --- .../dory/blitzar_metadata_table.rs | 507 +++++++++++------- 1 file changed, 299 insertions(+), 208 deletions(-) diff --git a/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs b/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs index 30be83197..7ca8cdef1 100644 --- a/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs +++ b/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs @@ -272,219 +272,325 @@ mod tests { use crate::base::math::decimal::Precision; use proof_of_sql_parser::posql_time::{PoSQLTimeUnit, PoSQLTimeZone}; + fn assert_blitzar_metadata( + committable_columns: &[CommittableColumn], + offset: usize, + expected_bit_table: &[u32], + expected_length_table: &[u32], + expected_scalars: &[u8], + ) { + let (bit_table, length_table, scalars) = + create_blitzar_metadata_tables(committable_columns, offset); + + assert_eq!( + bit_table, expected_bit_table, + "Bit table mismatch for offset {offset}" + ); + assert_eq!( + length_table, expected_length_table, + "Length table mismatch for offset {offset}" + ); + assert_eq!( + scalars, expected_scalars, + "Scalars mismatch for offset {offset}" + ); + } + #[test] - fn we_can_populate_blitzar_metadata_tables_with_empty_columns_with_full_column_offsets() { + fn we_can_populate_blitzar_metadata_tables_with_empty_columns_and_offset_that_fills_row() { let committable_columns = [CommittableColumn::BigInt(&[0; 0])]; let offsets = vec![ 0, 1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, ]; for &offset in &offsets { - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); - - assert!(bit_table.is_empty()); - assert!(length_table.is_empty()); - assert!(scalars.is_empty()); + assert_blitzar_metadata(&committable_columns, offset, &[], &[], &[]); } } #[test] - fn we_can_populate_blitzar_metadata_tables_with_empty_columns_and_an_offset_with_partial_column_offsets( + fn we_can_populate_blitzar_metadata_tables_with_empty_columns_and_offset_that_does_not_fill_row( ) { let committable_columns = [CommittableColumn::BigInt(&[0; 0])]; let offset = 3; - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); - - assert_eq!(bit_table, vec![64, 8]); - assert_eq!(length_table, vec![2, 2]); - assert_eq!( - scalars, - vec![0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + assert_blitzar_metadata( + &committable_columns, + offset, + &[64, 8], + &[2, 2], + &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], ); let offset = 5; - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); - - assert_eq!(bit_table, vec![64, 8]); - assert_eq!(length_table, vec![4, 4]); - assert_eq!( - scalars, - vec![ + assert_blitzar_metadata( + &committable_columns, + offset, + &[64, 8], + &[4, 4], + &[ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0 - ] + 0, 0, 0, 0, 0, 0, 0, 0, + ], ); let offset = 17; - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); - - assert_eq!(bit_table, vec![64, 8]); - assert_eq!(length_table, vec![8, 8]); - assert_eq!( - scalars, - vec![ + assert_blitzar_metadata( + &committable_columns, + offset, + &[64, 8], + &[8, 8], + &[ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - ] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ], ); let offset = 65; - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); - - assert_eq!(bit_table, vec![64, 8]); - assert_eq!(length_table, vec![16, 16]); - assert_eq!( - scalars, - vec![ + assert_blitzar_metadata( + &committable_columns, + offset, + &[64, 8], + &[16, 16], + &[ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0 - ] + 0, 0, 0, 0, + ], ); } #[test] fn we_can_populate_blitzar_metadata_tables_with_simple_column() { let committable_columns = [CommittableColumn::BigInt(&[1])]; - let offset = 0; - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); - assert_eq!(bit_table, vec![64, 8]); - assert_eq!(length_table, vec![1, 1]); - assert_eq!(scalars, vec![1, 0, 0, 0, 0, 0, 0, 128, 1]); + let offset = 0; + assert_blitzar_metadata( + &committable_columns, + offset, + &[64, 8], + &[1, 1], + &[1, 0, 0, 0, 0, 0, 0, 128, 1], + ); } #[test] fn we_can_populate_blitzar_metadata_tables_with_simple_column_and_offset() { let committable_columns = [CommittableColumn::BigInt(&[1])]; - let offset = 1; - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); - assert_eq!(bit_table, vec![64, 8]); - assert_eq!(length_table, vec![2, 2]); - assert_eq!( - scalars, - vec![0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 128, 1] + let offset = 1; + assert_blitzar_metadata( + &committable_columns, + offset, + &[64, 8], + &[2, 2], + &[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 128, 1], ); } #[test] fn we_can_populate_blitzar_metadata_tables_with_simple_column_and_non_trivial_offsets() { let committable_columns = [CommittableColumn::TinyInt(&[1])]; + let expected_bit_table = vec![8, 8]; let offset = 0; - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); - - assert_eq!(bit_table, vec![8, 8]); - assert_eq!(length_table, vec![1, 1]); - assert_eq!(scalars, vec![129, 1]); + assert_blitzar_metadata( + &committable_columns, + offset, + &expected_bit_table, + &[1, 1], + &[129, 1], + ); let offset = 1; - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); - - assert_eq!(bit_table, vec![8, 8]); - assert_eq!(length_table, vec![2, 2]); - assert_eq!(scalars, vec![0, 0, 129, 1]); + assert_blitzar_metadata( + &committable_columns, + offset, + &expected_bit_table, + &[2, 2], + &[0, 0, 129, 1], + ); let offset = 2; - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); - - assert_eq!(bit_table, vec![8, 8]); - assert_eq!(length_table, vec![2, 2]); - assert_eq!(scalars, vec![129, 1, 0, 0]); + assert_blitzar_metadata( + &committable_columns, + offset, + &expected_bit_table, + &[2, 2], + &[129, 1, 0, 0], + ); let offset = 3; - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); - - assert_eq!(bit_table, vec![8, 8]); - assert_eq!(length_table, vec![2, 2]); - assert_eq!(scalars, vec![0, 0, 129, 1]); - - let offset = 4; - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); - - assert_eq!(bit_table, vec![8, 8]); - assert_eq!(length_table, vec![4, 4]); - assert_eq!(scalars, vec![129, 1, 0, 0, 0, 0, 0, 0]); - - let offset = 5; - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); - - assert_eq!(bit_table, vec![8, 8]); - assert_eq!(length_table, vec![4, 4]); - assert_eq!(scalars, vec![0, 0, 129, 1, 0, 0, 0, 0]); - - let offset = 10; - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); - - assert_eq!(bit_table, vec![8, 8]); - assert_eq!(length_table, vec![4, 4]); - assert_eq!(scalars, vec![0, 0, 0, 0, 129, 1, 0, 0]); - - let offset = 16; - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); - - assert_eq!(bit_table, vec![8, 8]); - assert_eq!(length_table, vec![8, 8]); - assert_eq!( - scalars, - vec![129, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + assert_blitzar_metadata( + &committable_columns, + offset, + &expected_bit_table, + &[2, 2], + &[0, 0, 129, 1], ); + } - let offset = 20; - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); + #[test] + fn we_can_populate_blitzar_metadata_tables_with_simple_column_and_offsets_with_4_columns() { + let committable_columns = [CommittableColumn::TinyInt(&[1])]; - assert_eq!(bit_table, vec![8, 8]); - assert_eq!(length_table, vec![8, 8]); - assert_eq!( - scalars, - vec![0, 0, 0, 0, 0, 0, 0, 0, 129, 1, 0, 0, 0, 0, 0, 0] - ); + let offsets = vec![4, 8, 12]; + let expected_bit_table = vec![8, 8]; + let expected_length_table = vec![4, 4]; + for &offset in &offsets { + assert_blitzar_metadata( + &committable_columns, + offset, + &expected_bit_table, + &expected_length_table, + &[129, 1, 0, 0, 0, 0, 0, 0], + ); + + assert_blitzar_metadata( + &committable_columns, + offset + 1, + &expected_bit_table, + &expected_length_table, + &[0, 0, 129, 1, 0, 0, 0, 0], + ); + + assert_blitzar_metadata( + &committable_columns, + offset + 2, + &expected_bit_table, + &expected_length_table, + &[0, 0, 0, 0, 129, 1, 0, 0], + ); + + assert_blitzar_metadata( + &committable_columns, + offset + 3, + &expected_bit_table, + &expected_length_table, + &[0, 0, 0, 0, 0, 0, 129, 1], + ); + } + } - let offset = 50; - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); + #[test] + fn we_can_populate_blitzar_metadata_tables_with_simple_column_and_offsets_with_8_columns() { + let committable_columns = [CommittableColumn::TinyInt(&[1])]; - assert_eq!(bit_table, vec![8, 8]); - assert_eq!(length_table, vec![8, 8]); - assert_eq!( - scalars, - vec![0, 0, 0, 0, 129, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - ); + let offsets = vec![16, 24, 32, 40, 48, 56]; + let expected_bit_table = vec![8, 8]; + let expected_length_table = vec![8, 8]; + for &offset in &offsets { + assert_blitzar_metadata( + &committable_columns, + offset, + &expected_bit_table, + &expected_length_table, + &[129, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + ); + + assert_blitzar_metadata( + &committable_columns, + offset + 1, + &expected_bit_table, + &expected_length_table, + &[0, 0, 129, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + ); + + assert_blitzar_metadata( + &committable_columns, + offset + 2, + &expected_bit_table, + &expected_length_table, + &[0, 0, 0, 0, 129, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + ); + + assert_blitzar_metadata( + &committable_columns, + offset + 3, + &expected_bit_table, + &expected_length_table, + &[0, 0, 0, 0, 0, 0, 129, 1, 0, 0, 0, 0, 0, 0, 0, 0], + ); + + assert_blitzar_metadata( + &committable_columns, + offset + 4, + &expected_bit_table, + &expected_length_table, + &[0, 0, 0, 0, 0, 0, 0, 0, 129, 1, 0, 0, 0, 0, 0, 0], + ); + + assert_blitzar_metadata( + &committable_columns, + offset + 5, + &expected_bit_table, + &expected_length_table, + &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 1, 0, 0, 0, 0], + ); + + assert_blitzar_metadata( + &committable_columns, + offset + 6, + &expected_bit_table, + &expected_length_table, + &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 1, 0, 0], + ); + + assert_blitzar_metadata( + &committable_columns, + offset + 7, + &expected_bit_table, + &expected_length_table, + &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 1], + ); + } + } - let offset = 64; - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); + #[test] + fn we_can_populate_blitzar_metadata_tables_with_simple_column_and_offsets_with_16_columns() { + let committable_columns = [CommittableColumn::TinyInt(&[1])]; - assert_eq!(bit_table, vec![8, 8]); - assert_eq!(length_table, vec![16, 16]); - assert_eq!( - scalars, - vec![ - 129, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0 - ] - ); + let offsets = vec![64, 80, 96, 112]; + let expected_bit_table = vec![8, 8]; + let expected_length_table = vec![16, 16]; + + for &offset in &offsets { + assert_blitzar_metadata( + &committable_columns, + offset, + &expected_bit_table, + &expected_length_table, + &[ + 129, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + ], + ); + + assert_blitzar_metadata( + &committable_columns, + offset + 8, + &expected_bit_table, + &expected_length_table, + &[ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + ], + ); + + assert_blitzar_metadata( + &committable_columns, + offset + 15, + &expected_bit_table, + &expected_length_table, + &[ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 129, 1, + ], + ); + } } #[test] @@ -503,27 +609,21 @@ mod tests { ]; let offset = 0; - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); - assert_eq!( - bit_table, - vec![8, 16, 32, 64, 128, 256, 256, 256, 64, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8] - ); - - assert_eq!( - length_table, - vec![1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] - ); - assert_eq!( - scalars, - vec![ + assert_blitzar_metadata( + &committable_columns, + offset, + &[ + 8, 16, 32, 64, 128, 256, 256, 256, 64, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + ], + &[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + &[ 129, 2, 128, 3, 0, 0, 128, 4, 0, 0, 0, 0, 0, 0, 128, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 128, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 - ] + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + ], ); } @@ -542,34 +642,31 @@ mod tests { CommittableColumn::Boolean(&[true]), ]; - let offset = 1; - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); - assert_eq!( - bit_table, - vec![8, 16, 32, 64, 128, 256, 256, 256, 64, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8] - ); - - assert_eq!( - length_table, - vec![2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2] - ); - assert_eq!( - scalars, - vec![ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 129, 2, 128, 3, 0, 0, 128, 4, 0, 0, 0, 0, 0, 0, 128, 5, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, - 0, 0, 0, 0, 128, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 - ] - ); + let offsets = vec![1, 3]; + for &offset in &offsets { + assert_blitzar_metadata( + &committable_columns, + offset, + &[ + 8, 16, 32, 64, 128, 256, 256, 256, 64, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + ], + &[2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], + &[ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 2, 128, 3, 0, 0, 128, 4, + 0, 0, 0, 0, 0, 0, 128, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 6, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 128, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, + ], + ); + } } #[test] @@ -588,20 +685,14 @@ mod tests { ]; let offset = 2; - let (bit_table, length_table, scalars) = - create_blitzar_metadata_tables(&committable_columns, offset); - assert_eq!( - bit_table, - vec![8, 16, 32, 64, 128, 256, 256, 256, 64, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8] - ); - - assert_eq!( - length_table, - vec![2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2] - ); - assert_eq!( - scalars, - vec![ + assert_blitzar_metadata( + &committable_columns, + offset, + &[ + 8, 16, 32, 64, 128, 256, 256, 256, 64, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + ], + &[2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], + &[ 129, 2, 128, 3, 0, 0, 128, 4, 0, 0, 0, 0, 0, 0, 128, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -612,8 +703,8 @@ mod tests { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - ] + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ], ); } } From 33e9f22b43892b1a2f541804e13038dc9f2c7745 Mon Sep 17 00:00:00 2001 From: Jacob Trombetta Date: Mon, 21 Oct 2024 21:13:15 -0400 Subject: [PATCH 5/6] chore: update test format for readability --- .../proof_primitive/dory/blitzar_metadata_table.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs b/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs index 7ca8cdef1..52d3770e4 100644 --- a/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs +++ b/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs @@ -394,6 +394,7 @@ mod tests { #[test] fn we_can_populate_blitzar_metadata_tables_with_simple_column_and_non_trivial_offsets() { let committable_columns = [CommittableColumn::TinyInt(&[1])]; + let expected_bit_table = vec![8, 8]; let offset = 0; @@ -437,9 +438,10 @@ mod tests { fn we_can_populate_blitzar_metadata_tables_with_simple_column_and_offsets_with_4_columns() { let committable_columns = [CommittableColumn::TinyInt(&[1])]; - let offsets = vec![4, 8, 12]; let expected_bit_table = vec![8, 8]; let expected_length_table = vec![4, 4]; + + let offsets = vec![4, 8, 12]; for &offset in &offsets { assert_blitzar_metadata( &committable_columns, @@ -479,9 +481,10 @@ mod tests { fn we_can_populate_blitzar_metadata_tables_with_simple_column_and_offsets_with_8_columns() { let committable_columns = [CommittableColumn::TinyInt(&[1])]; - let offsets = vec![16, 24, 32, 40, 48, 56]; let expected_bit_table = vec![8, 8]; let expected_length_table = vec![8, 8]; + + let offsets = vec![16, 24, 32, 40, 48, 56]; for &offset in &offsets { assert_blitzar_metadata( &committable_columns, @@ -553,10 +556,10 @@ mod tests { fn we_can_populate_blitzar_metadata_tables_with_simple_column_and_offsets_with_16_columns() { let committable_columns = [CommittableColumn::TinyInt(&[1])]; - let offsets = vec![64, 80, 96, 112]; let expected_bit_table = vec![8, 8]; let expected_length_table = vec![16, 16]; - + + let offsets = vec![64, 80, 96, 112]; for &offset in &offsets { assert_blitzar_metadata( &committable_columns, From 2cb0d0b0fc51cb7d7f34503591a0898bea9a3124 Mon Sep 17 00:00:00 2001 From: Jacob Trombetta Date: Mon, 21 Oct 2024 23:14:24 -0400 Subject: [PATCH 6/6] chore: cargo format correction --- .../src/proof_primitive/dory/blitzar_metadata_table.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs b/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs index 52d3770e4..72e2ca8cb 100644 --- a/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs +++ b/crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs @@ -558,7 +558,7 @@ mod tests { let expected_bit_table = vec![8, 8]; let expected_length_table = vec![16, 16]; - + let offsets = vec![64, 80, 96, 112]; for &offset in &offsets { assert_blitzar_metadata(