Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove io_optimized / sbq_speedup storage mode #195

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Remove io_optimized / sbq_speedup storage mode #195

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6f8c9d1
Remove io_optimized storage type, which does not perform well and is …
tjgreen42 Jan 21, 2025
f7b4a86
Remove io_optimized storage type, which does not perform well and is …
tjgreen42 Jan 21, 2025
ec182e0
Revert "Remove io_optimized storage type, which does not perform well…
tjgreen42 Jan 21, 2025
d1b3d15
Prefer 'memory_optimized' and stop advertising 'bq_compression'
tjgreen42 Jan 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions pgvectorscale/src/access_method/build.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ unsafe fn aminsert_internal(
&mut stats,
);
}
StorageType::SbqSpeedup | StorageType::SbqCompression => {
StorageType::SbqCompression => {
let bq = SbqSpeedupStorage::load_for_insert(
&heap_relation,
&index_relation,
Expand Down Expand Up @@ -282,7 +282,7 @@ fn do_heap_scan<'a>(

finalize_index_build(&mut plain, &mut bs, write_stats)
}
StorageType::SbqSpeedup | StorageType::SbqCompression => {
StorageType::SbqCompression => {
let mut bq =
SbqSpeedupStorage::new_for_build(index_relation, heap_relation, &meta_page);

Expand Down
30 changes: 3 additions & 27 deletions pgvectorscale/src/access_method/meta_page.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ use super::options::{
NUM_DIMENSIONS_DEFAULT_SENTINEL, NUM_NEIGHBORS_DEFAULT_SENTINEL,
SBQ_NUM_BITS_PER_DIMENSION_DEFAULT_SENTINEL,
};
use super::sbq::SbqNode;
use super::stats::StatsNodeModify;
use super::storage::StorageType;

Expand Down Expand Up @@ -134,16 +133,6 @@ impl MetaPage {
self.bq_num_bits_per_dimension
}

pub fn get_num_dimensions_for_neighbors(&self) -> u32 {
match StorageType::from_u8(self.storage_type) {
StorageType::Plain => {
error!("get_num_dimensions_for_neighbors should not be called for Plain storage")
}
StorageType::SbqSpeedup => self.num_dimensions_to_index,
StorageType::SbqCompression => 0,
}
}

/// Maximum number of neigbors per node. Given that we pre-allocate
/// these many slots for each node, this cannot change after the graph is built.
pub fn get_num_neighbors(&self) -> u32 {
Expand Down Expand Up @@ -189,24 +178,15 @@ impl MetaPage {

match self.get_storage_type() {
StorageType::Plain => None,
StorageType::SbqSpeedup | StorageType::SbqCompression => Some(self.quantizer_metadata),
StorageType::SbqCompression => Some(self.quantizer_metadata),
}
}

fn calculate_num_neighbors(
num_dimensions: u32,
num_bits_per_dimension: u8,
opt: &PgBox<TSVIndexOptions>,
) -> u32 {
fn calculate_num_neighbors(opt: &PgBox<TSVIndexOptions>) -> u32 {
let num_neighbors = (*opt).get_num_neighbors();
if num_neighbors == NUM_NEIGHBORS_DEFAULT_SENTINEL {
match (*opt).get_storage_type() {
StorageType::Plain => 50,
StorageType::SbqSpeedup => SbqNode::get_default_num_neighbors(
num_dimensions as usize,
num_dimensions as usize,
num_bits_per_dimension,
) as u32,
StorageType::SbqCompression => 50,
}
} else {
Expand Down Expand Up @@ -262,11 +242,7 @@ impl MetaPage {
num_dimensions,
num_dimensions_to_index,
storage_type: (*opt).get_storage_type() as u8,
num_neighbors: Self::calculate_num_neighbors(
num_dimensions,
bq_num_bits_per_dimension,
&opt,
),
num_neighbors: Self::calculate_num_neighbors(&opt),
bq_num_bits_per_dimension,
search_list_size: opt.search_list_size,
max_alpha: opt.max_alpha,
Expand Down
24 changes: 1 addition & 23 deletions pgvectorscale/src/access_method/options.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ pub unsafe fn init() {
pg_sys::add_string_reloption(
RELOPT_KIND_TSV,
"storage_layout".as_pg_cstr(),
"Storage layout: either memory_optimized, io_optimized, or plain".as_pg_cstr(),
"Storage layout: either memory_optimized or plain".as_pg_cstr(),
super::storage::DEFAULT_STORAGE_TYPE_STR.as_pg_cstr(),
Some(validate_storage_layout),
pg_sys::AccessExclusiveLock as pg_sys::LOCKMODE,
Expand Down Expand Up @@ -298,28 +298,6 @@ mod tests {
Ok(())
}

#[pg_test]
unsafe fn test_index_options_bq() -> spi::Result<()> {
Spi::run(
"CREATE TABLE test(encoding vector(3));
CREATE INDEX idxtest
ON test
USING diskann(encoding)
WITH (storage_layout = io_optimized);",
)?;

let index_oid =
Spi::get_one::<pg_sys::Oid>("SELECT 'idxtest'::regclass::oid")?.expect("oid was null");
let indexrel = PgRelation::from_pg(pg_sys::RelationIdGetRelation(index_oid));
let options = TSVIndexOptions::from_relation(&indexrel);
assert_eq!(options.get_num_neighbors(), NUM_NEIGHBORS_DEFAULT_SENTINEL);
assert_eq!(options.search_list_size, 100);
assert_eq!(options.max_alpha, DEFAULT_MAX_ALPHA);
assert_eq!(options.num_dimensions, NUM_DIMENSIONS_DEFAULT_SENTINEL);
assert_eq!(options.get_storage_type(), StorageType::SbqSpeedup);
Ok(())
}

#[pg_test]
unsafe fn test_index_options_plain() -> spi::Result<()> {
Spi::run(
Expand Down
Loading
Loading