Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove io_optimized / sbq_speedup storage mode #195

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Revert "Remove io_optimized storage type, which does not perform well…
… and is unused"

This reverts commit f7b4a86.
tjgreen42 committed Jan 21, 2025

Verified

This commit was signed with the committer’s verified signature.
paescuj Pascal Jufer
commit ec182e0ef3ed1a76e505e0deefb23bc3680374f7
4 changes: 2 additions & 2 deletions pgvectorscale/src/access_method/build.rs
Original file line number Diff line number Diff line change
@@ -198,7 +198,7 @@ unsafe fn aminsert_internal(
&mut stats,
);
}
StorageType::SbqSpeedup | StorageType::SbqCompression => {
StorageType::SbqCompression => {
let bq = SbqSpeedupStorage::load_for_insert(
&heap_relation,
&index_relation,
@@ -282,7 +282,7 @@ fn do_heap_scan<'a>(

finalize_index_build(&mut plain, &mut bs, write_stats)
}
StorageType::SbqSpeedup | StorageType::SbqCompression => {
StorageType::SbqCompression => {
let mut bq =
SbqSpeedupStorage::new_for_build(index_relation, heap_relation, &meta_page);

30 changes: 3 additions & 27 deletions pgvectorscale/src/access_method/meta_page.rs
Original file line number Diff line number Diff line change
@@ -13,7 +13,6 @@ use super::options::{
NUM_DIMENSIONS_DEFAULT_SENTINEL, NUM_NEIGHBORS_DEFAULT_SENTINEL,
SBQ_NUM_BITS_PER_DIMENSION_DEFAULT_SENTINEL,
};
use super::sbq::SbqNode;
use super::stats::StatsNodeModify;
use super::storage::StorageType;

@@ -134,16 +133,6 @@ impl MetaPage {
self.bq_num_bits_per_dimension
}

pub fn get_num_dimensions_for_neighbors(&self) -> u32 {
match StorageType::from_u8(self.storage_type) {
StorageType::Plain => {
error!("get_num_dimensions_for_neighbors should not be called for Plain storage")
}
StorageType::SbqSpeedup => self.num_dimensions_to_index,
StorageType::SbqCompression => 0,
}
}

/// Maximum number of neigbors per node. Given that we pre-allocate
/// these many slots for each node, this cannot change after the graph is built.
pub fn get_num_neighbors(&self) -> u32 {
@@ -189,24 +178,15 @@ impl MetaPage {

match self.get_storage_type() {
StorageType::Plain => None,
StorageType::SbqSpeedup | StorageType::SbqCompression => Some(self.quantizer_metadata),
StorageType::SbqCompression => Some(self.quantizer_metadata),
}
}

fn calculate_num_neighbors(
num_dimensions: u32,
num_bits_per_dimension: u8,
opt: &PgBox<TSVIndexOptions>,
) -> u32 {
fn calculate_num_neighbors(opt: &PgBox<TSVIndexOptions>) -> u32 {
let num_neighbors = (*opt).get_num_neighbors();
if num_neighbors == NUM_NEIGHBORS_DEFAULT_SENTINEL {
match (*opt).get_storage_type() {
StorageType::Plain => 50,
StorageType::SbqSpeedup => SbqNode::get_default_num_neighbors(
num_dimensions as usize,
num_dimensions as usize,
num_bits_per_dimension,
) as u32,
StorageType::SbqCompression => 50,
}
} else {
@@ -262,11 +242,7 @@ impl MetaPage {
num_dimensions,
num_dimensions_to_index,
storage_type: (*opt).get_storage_type() as u8,
num_neighbors: Self::calculate_num_neighbors(
num_dimensions,
bq_num_bits_per_dimension,
&opt,
),
num_neighbors: Self::calculate_num_neighbors(&opt),
bq_num_bits_per_dimension,
search_list_size: opt.search_list_size,
max_alpha: opt.max_alpha,
22 changes: 0 additions & 22 deletions pgvectorscale/src/access_method/options.rs
Original file line number Diff line number Diff line change
@@ -298,28 +298,6 @@ mod tests {
Ok(())
}

#[pg_test]
unsafe fn test_index_options_bq() -> spi::Result<()> {
Spi::run(
"CREATE TABLE test(encoding vector(3));
CREATE INDEX idxtest
ON test
USING diskann(encoding)
WITH (storage_layout = io_optimized);",
)?;

let index_oid =
Spi::get_one::<pg_sys::Oid>("SELECT 'idxtest'::regclass::oid")?.expect("oid was null");
let indexrel = PgRelation::from_pg(pg_sys::RelationIdGetRelation(index_oid));
let options = TSVIndexOptions::from_relation(&indexrel);
assert_eq!(options.get_num_neighbors(), NUM_NEIGHBORS_DEFAULT_SENTINEL);
assert_eq!(options.search_list_size, 100);
assert_eq!(options.max_alpha, DEFAULT_MAX_ALPHA);
assert_eq!(options.num_dimensions, NUM_DIMENSIONS_DEFAULT_SENTINEL);
assert_eq!(options.get_storage_type(), StorageType::SbqSpeedup);
Ok(())
}

#[pg_test]
unsafe fn test_index_options_plain() -> spi::Result<()> {
Spi::run(
267 changes: 28 additions & 239 deletions pgvectorscale/src/access_method/sbq.rs

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion pgvectorscale/src/access_method/scan.rs
Original file line number Diff line number Diff line change
@@ -67,7 +67,7 @@ impl TSVScanState {
TSVResponseIterator::new(&bq, index, query, search_list_size, meta_page, stats);
StorageState::Plain(it)
}
StorageType::SbqSpeedup | StorageType::SbqCompression => {
StorageType::SbqCompression => {
let mut stats = QuantizerStats::new();
let quantizer = unsafe { SbqMeans::load(index, &meta_page, &mut stats) };
let bq = SbqSpeedupStorage::load_for_search(index, heap, &quantizer, &meta_page);
6 changes: 2 additions & 4 deletions pgvectorscale/src/access_method/storage.rs
Original file line number Diff line number Diff line change
@@ -131,7 +131,7 @@ pub trait Storage {
#[derive(PartialEq, Debug)]
pub enum StorageType {
Plain = 0,
SbqSpeedup = 1,
// R.I.P. SbqSpeedup = 1,
SbqCompression = 2,
}

@@ -141,7 +141,6 @@ impl StorageType {
pub fn from_u8(value: u8) -> Self {
match value {
0 => StorageType::Plain,
1 => StorageType::SbqSpeedup,
2 => StorageType::SbqCompression,
_ => panic!("Invalid storage type"),
}
@@ -150,10 +149,9 @@ impl StorageType {
pub fn from_str(value: &str) -> Self {
match value.to_lowercase().as_str() {
"plain" => StorageType::Plain,
"bq_speedup" | "io_optimized" => StorageType::SbqSpeedup,
"bq_compression" | "memory_optimized" => StorageType::SbqCompression,
_ => panic!(
"Invalid storage type. Must be one of 'plain', 'bq_speedup', 'bq_compression'"
"Invalid storage type. Must be either 'plain' or 'bq_compression' aka 'memory_optimized'"
),
}
}
2 changes: 1 addition & 1 deletion pgvectorscale/src/access_method/vacuum.rs
Original file line number Diff line number Diff line change
@@ -40,7 +40,7 @@ pub extern "C" fn ambulkdelete(
let meta_page = MetaPage::fetch(&index_relation);
let storage = meta_page.get_storage_type();
match storage {
StorageType::SbqSpeedup | StorageType::SbqCompression => {
StorageType::SbqCompression => {
bulk_delete_for_storage::<SbqSpeedupStorage>(
&index_relation,
nblocks,