From 0124ea8cd768785526ee5c1981c4ae7485a3ef4a Mon Sep 17 00:00:00 2001 From: Jay White Date: Fri, 27 Sep 2024 09:23:09 -0400 Subject: [PATCH 1/2] feat: add `std` feature flag --- .github/workflows/lint-and-test.yml | 16 ++++++++--- Cargo.toml | 27 +++++++++---------- crates/proof-of-sql-parser/src/lib.rs | 1 + crates/proof-of-sql/Cargo.toml | 12 +++++---- .../proof-of-sql/src/base/commitment/mod.rs | 1 + crates/proof-of-sql/src/lib.rs | 1 + .../src/sql/parse/query_expr_tests.rs | 7 +++-- .../postprocessing/group_by_postprocessing.rs | 27 ++++++++++++------- 8 files changed, 57 insertions(+), 35 deletions(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 3c3c90a6f..63f3b4c2b 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -49,6 +49,14 @@ jobs: run: cargo check -p proof-of-sql --no-default-features --features="test" - name: Run cargo check (proof-of-sql) (just "blitzar" feature) run: cargo check -p proof-of-sql --no-default-features --features="blitzar" + - name: Run cargo check (proof-of-sql) (just "arrow" feature) + run: cargo check -p proof-of-sql --no-default-features --features="arrow" + - name: Run cargo check (proof-of-sql) (just "rayon" feature) + run: cargo check -p proof-of-sql --no-default-features --features="rayon" + - name: Run cargo check (proof-of-sql) (just "perf" feature) + run: cargo check -p proof-of-sql --no-default-features --features="perf" + - name: Run cargo check (proof-of-sql) (just "std" feature) + run: cargo check -p proof-of-sql --no-default-features --features="std" test: name: Test Suite @@ -83,10 +91,10 @@ jobs: run: cargo test -p proof-of-sql --no-run --no-default-features --features="arrow" - name: Dry run cargo test (proof-of-sql) (blitzar feature only) run: cargo test -p proof-of-sql --no-run --no-default-features --features="blitzar" - - name: Dry run cargo test (proof-of-sql) (no features) - run: cargo test -p proof-of-sql --no-run --no-default-features - - name: Run cargo test (proof primitives - Dory) (no features - i.e. not using blitzar) - run: cargo test proof_primitive::dory::dory_compute_commitments_test --no-default-features + - name: Dry run cargo test (proof-of-sql) (std feature only) + run: cargo test -p proof-of-sql --no-run --no-default-features --features="std" + - name: Run cargo test (proof primitives - Dory) (std feature only - i.e. not using blitzar) + run: cargo test proof_primitive::dory::dory_compute_commitments_test --no-default-features --features="std" - name: Run hello_world example run: cargo run --example hello_world --features="blitzar test" - name: Run posql_db example diff --git a/Cargo.toml b/Cargo.toml index 3c03178ac..bcf02605f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,26 +10,26 @@ version = "0.0.0" # DO NOT CHANGE THIS LINE! This will be automatically updated license-file = "LICENSE" [workspace.dependencies] -ahash = { version = "0.8.11", default-features = false, features = ["runtime-rng"] } +ahash = { version = "0.8.11", default-features = false } # alloy-primitives = { version = "0.8.1" } # alloy-sol-types = { version = "0.8.1" } ark-bls12-381 = { version = "0.4.0" } ark-curve25519 = { version = "0.4.0" } -ark-ec = { version = "0.4.0", features = [ "parallel" ] } -ark-ff = { version = "0.4.0", features = [ "parallel" ] } -ark-poly = { version = "0.4.0", features = [ "parallel" ] } +ark-ec = { version = "0.4.0" } +ark-ff = { version = "0.4.0" } +ark-poly = { version = "0.4.0" } ark-serialize = { version = "0.4.0" } -ark-std = { version = "0.4.0", features = [ "parallel" ] } +ark-std = { version = "0.4.0", default-features = false } arrayvec = { version = "0.7", default-features = false } arrow = { version = "51.0" } arrow-csv = { version = "51.0" } bit-iter = { version = "1.1.1" } bigdecimal = { version = "0.4.5", default-features = false, features = ["serde"] } -blake3 = { version = "1.3.3" } +blake3 = { version = "1.3.3", default-features = false } blitzar = { version = "3.1.0" } bumpalo = { version = "3.11.0" } bytemuck = {version = "1.16.3", features = ["derive"]} -byte-slice-cast = { version = "1.2.1" } +byte-slice-cast = { version = "1.2.1", default-features = false } clap = { version = "4.5.4" } criterion = { version = "0.5.1" } chrono = { version = "0.4.38", default-features = false } @@ -38,12 +38,11 @@ derive_more = { version = "0.99" } flexbuffers = { version = "2.0.0" } # forge-script = { git = "https://github.com/foundry-rs/foundry", tag = "nightly-bf1a39980532f76cd76fd87ee32661180f606435" } indexmap = { version = "2.1", default-features = false } -itertools = { version = "0.13.0" } +itertools = { version = "0.13.0", default-features = false, features = ["use_alloc"] } lalrpop = { version = "0.21.0" } -lalrpop-util = { version = "0.20.0", default-features = false } -lazy_static = { version = "1.4.0" } +lalrpop-util = { version = "0.21.0", default-features = false } merlin = { version = "2" } -num-traits = { version = "0.2" } +num-traits = { version = "0.2", default-features = false } num-bigint = { version = "0.4.4", default-features = false } opentelemetry = { version = "0.23.0" } opentelemetry-jaeger = { version = "0.20.0" } @@ -54,11 +53,11 @@ rand = { version = "0.8", default-features = false } rand_core = { version = "0.6", default-features = false } rayon = { version = "1.5" } serde = { version = "1", default-features = false } -serde_json = { version = "1" } -snafu = { version = "0.8.4", default-features = false, features = ["std"] } +serde_json = { version = "1", default-features = false, features = ["alloc"] } +snafu = { version = "0.8.4", default-features = false } tiny-keccak = { version = "2.0.2", features = [ "keccak" ] } # tokio = { version = "1.39.3" } -tracing = { version = "0.1.36" } +tracing = { version = "0.1.36", default-features = false } tracing-opentelemetry = { version = "0.22.0" } tracing-subscriber = { version = "0.3.0" } wasm-bindgen = { version = "0.2.92" } diff --git a/crates/proof-of-sql-parser/src/lib.rs b/crates/proof-of-sql-parser/src/lib.rs index 1397066c4..6702f5479 100644 --- a/crates/proof-of-sql-parser/src/lib.rs +++ b/crates/proof-of-sql-parser/src/lib.rs @@ -1,4 +1,5 @@ #![doc = include_str!("../README.md")] +#![no_std] extern crate alloc; /// Module for handling an intermediate decimal type received from the lexer. diff --git a/crates/proof-of-sql/Cargo.toml b/crates/proof-of-sql/Cargo.toml index bbf2b117e..eb13d6a6c 100644 --- a/crates/proof-of-sql/Cargo.toml +++ b/crates/proof-of-sql/Cargo.toml @@ -35,7 +35,6 @@ chrono = { workspace = true, features = ["serde"] } derive_more = { workspace = true } indexmap = { workspace = true, features = ["serde"] } itertools = { workspace = true } -lazy_static = { workspace = true } merlin = { workspace = true, optional = true } num-traits = { workspace = true } num-bigint = { workspace = true, default-features = false } @@ -74,10 +73,13 @@ flexbuffers = { workspace = true } development = ["arrow-csv"] [features] -default = ["arrow", "blitzar", "rayon"] -arrow = ["dep:arrow"] -blitzar = ["dep:blitzar", "dep:merlin"] -test = ["dep:rand"] +default = ["arrow", "perf"] +arrow = ["dep:arrow", "std"] +blitzar = ["dep:blitzar", "dep:merlin", "std"] +test = ["dep:rand", "std"] +perf = ["blitzar", "rayon", "ark-ec/parallel", "ark-poly/parallel", "ark-ff/asm"] +rayon = ["dep:rayon", "std"] +std = ["snafu/std"] [lints] workspace = true diff --git a/crates/proof-of-sql/src/base/commitment/mod.rs b/crates/proof-of-sql/src/base/commitment/mod.rs index 70c128521..6231061f0 100644 --- a/crates/proof-of-sql/src/base/commitment/mod.rs +++ b/crates/proof-of-sql/src/base/commitment/mod.rs @@ -1,5 +1,6 @@ //! Types for creation and utilization of cryptographic commitments to proof-of-sql data. use crate::base::scalar::Scalar; +use alloc::vec::Vec; #[cfg(feature = "blitzar")] pub use blitzar::{ compute::{init_backend, init_backend_with_config, BackendConfig}, diff --git a/crates/proof-of-sql/src/lib.rs b/crates/proof-of-sql/src/lib.rs index d1b4112b8..ab3c2e6ab 100644 --- a/crates/proof-of-sql/src/lib.rs +++ b/crates/proof-of-sql/src/lib.rs @@ -1,4 +1,5 @@ #![doc = include_str!("../README.md")] +#![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; pub mod base; diff --git a/crates/proof-of-sql/src/sql/parse/query_expr_tests.rs b/crates/proof-of-sql/src/sql/parse/query_expr_tests.rs index fe49e9817..e80d9dee4 100644 --- a/crates/proof-of-sql/src/sql/parse/query_expr_tests.rs +++ b/crates/proof-of-sql/src/sql/parse/query_expr_tests.rs @@ -2,7 +2,7 @@ use super::ConversionError; use crate::{ base::{ database::{ColumnType, TableRef, TestSchemaAccessor}, - map::{indexmap, IndexMap}, + map::{indexmap, IndexMap, IndexSet}, }, sql::{ parse::QueryExpr, @@ -1881,7 +1881,10 @@ fn select_group_and_order_by_preserve_the_column_order_reference() { let (t, accessor) = get_test_accessor(); let base_cols: [&str; N] = ["i", "i0", "i1", "s"]; // sorted because of `select: [cols = ... ]` let base_ordering = [Asc, Desc, Asc, Desc]; - for (idx, perm_cols) in base_cols.into_iter().permutations(N).unique().enumerate() { + for (idx, perm_cols) in IndexSet::from_iter(base_cols.into_iter().permutations(N)) + .into_iter() + .enumerate() + { let perm_col_plans = perm_cols .iter() .sorted() diff --git a/crates/proof-of-sql/src/sql/postprocessing/group_by_postprocessing.rs b/crates/proof-of-sql/src/sql/postprocessing/group_by_postprocessing.rs index 6fb4304c7..4c8935ef8 100644 --- a/crates/proof-of-sql/src/sql/postprocessing/group_by_postprocessing.rs +++ b/crates/proof-of-sql/src/sql/postprocessing/group_by_postprocessing.rs @@ -12,7 +12,6 @@ use proof_of_sql_parser::{ Identifier, }; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; /// A group by expression #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -158,7 +157,7 @@ impl GroupByPostprocessing { ) }) .collect::>>()?; - let group_by_identifiers = by_ids.into_iter().unique().collect(); + let group_by_identifiers = Vec::from_iter(IndexSet::from_iter(by_ids)); Ok(Self { remainder_exprs, group_by_identifiers, @@ -190,14 +189,22 @@ impl PostprocessingStep for GroupByPostprocessing { fn apply(&self, owned_table: OwnedTable) -> PostprocessingResult> { // First evaluate all the aggregated columns let alloc = Bump::new(); - let evaluated_columns: HashMap)>> = - self.aggregation_exprs - .iter() - .map(|(agg_op, expr, id)| -> PostprocessingResult<_> { - let evaluated_owned_column = owned_table.evaluate(expr)?; - Ok((*agg_op, (*id, evaluated_owned_column))) - }) - .process_results(|iter| iter.into_group_map())?; + let evaluated_columns = self + .aggregation_exprs + .iter() + .map(|(agg_op, expr, id)| -> PostprocessingResult<_> { + let evaluated_owned_column = owned_table.evaluate(expr)?; + Ok((*agg_op, (*id, evaluated_owned_column))) + }) + .process_results(|iter| { + iter.fold( + IndexMap::<_, Vec<_>>::default(), + |mut lookup, (key, val)| { + lookup.entry(key).or_default().push(val); + lookup + }, + ) + })?; // Next actually do the GROUP BY let group_by_ins = self .group_by_identifiers From df91594013f549b4c680f366853390ff1b0948d1 Mon Sep 17 00:00:00 2001 From: Jay White Date: Fri, 27 Sep 2024 10:13:07 -0400 Subject: [PATCH 2/2] ci: add hacky CI to ensure `no_std` works This patches `lalrpop` with a change that makes it `no_std` compatible. --- .github/workflows/lint-and-test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 63f3b4c2b..80a30d729 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -57,6 +57,11 @@ jobs: run: cargo check -p proof-of-sql --no-default-features --features="perf" - name: Run cargo check (proof-of-sql) (just "std" feature) run: cargo check -p proof-of-sql --no-default-features --features="std" + - name: Run cargo check (proof-of-sql-parser) with no_std target. This requires a lalrpop patch. + run: | + rustup target add thumbv7em-none-eabi + printf '\n[patch.crates-io]\nlalrpop = { git = "https://github.com/lalrpop/lalrpop", rev = "173597c" }\nlalrpop-util = { git = "https://github.com/lalrpop/lalrpop", rev = "173597c" }\n' >> Cargo.toml + cargo check -p proof-of-sql-parser --target thumbv7em-none-eabi test: name: Test Suite