Skip to content

Commit

Permalink
Use rowid for virtual tables
Browse files Browse the repository at this point in the history
  • Loading branch information
ankane committed Oct 6, 2024
1 parent d66c9b7 commit 2909522
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 6 deletions.
12 changes: 10 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -307,14 +307,12 @@ class AddEmbeddingToItems < ActiveRecord::Migration[7.2]
# Rails < 8
execute <<~SQL
CREATE VIRTUAL TABLE items USING vec0(
id integer PRIMARY KEY AUTOINCREMENT NOT NULL,
embedding float[3] distance_metric=L2
)
SQL

# Rails 8+
create_virtual_table :items, :vec0, [
"id integer PRIMARY KEY AUTOINCREMENT NOT NULL",
"embedding float[3] distance_metric=L2"
]
end
Expand All @@ -323,6 +321,16 @@ end

Use `distance_metric=cosine` for cosine distance

Create a model with `rowid` as the primary key

```ruby
class Item < ApplicationRecord
self.primary_key = "rowid"

has_neighbors :embedding, dimensions: 3
end
```

Get the nearest neighbors

```ruby
Expand Down
2 changes: 2 additions & 0 deletions test/sqlite_virtual_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ def test_euclidean
create_items(SqliteVecItem, :embedding)

relation = SqliteVecItem.where("embedding MATCH ?", [1, 1, 1].to_s).order(:distance).limit(3)
assert_equal [1, 3, 2], relation.all.map(&:id)
assert_equal [1, 3, 2], relation.pluck(:rowid)
assert_elements_in_delta [0, 1, Math.sqrt(3)], relation.pluck(:distance)
assert_match "SCAN vec_items VIRTUAL TABLE INDEX", relation.explain.inspect

Expand Down
6 changes: 2 additions & 4 deletions test/support/sqlite.rb
Original file line number Diff line number Diff line change
Expand Up @@ -15,27 +15,23 @@ class SqliteRecord < ActiveRecord::Base

if ActiveRecord::VERSION::MAJOR >= 8
create_virtual_table :vec_items, :vec0, [
"id integer PRIMARY KEY AUTOINCREMENT NOT NULL",
"embedding float[3] distance_metric=L2"
]
else
execute <<~SQL
CREATE VIRTUAL TABLE vec_items USING vec0(
id integer PRIMARY KEY AUTOINCREMENT NOT NULL,
embedding float[3] distance_metric=L2
)
SQL
end

if ActiveRecord::VERSION::MAJOR >= 8
create_virtual_table :cosine_items, :vec0, [
"id integer PRIMARY KEY AUTOINCREMENT NOT NULL",
"embedding float[3] distance_metric=cosine"
]
else
execute <<~SQL
CREATE VIRTUAL TABLE cosine_items USING vec0(
id integer PRIMARY KEY AUTOINCREMENT NOT NULL,
embedding float[3] distance_metric=cosine
)
SQL
Expand All @@ -51,11 +47,13 @@ class SqliteItem < SqliteRecord

class SqliteVecItem < SqliteRecord
has_neighbors :embedding, dimensions: 3
self.primary_key = "rowid"
self.table_name = "vec_items"
end

class SqliteCosineItem < SqliteRecord
has_neighbors :embedding, dimensions: 3
self.primary_key = "rowid"
self.table_name = "cosine_items"
end

Expand Down

0 comments on commit 2909522

Please sign in to comment.