diff --git a/scripts/concat.mjs b/scripts/concat.mjs index 6a7b2c1..d43d702 100644 --- a/scripts/concat.mjs +++ b/scripts/concat.mjs @@ -5,7 +5,7 @@ import { fileURLToPath } from 'url' const instructions = 'The above code was taken from my codebase at https://github.com/jointhealliance/bgent.' // Patterns to ignore -const ignorePatterns = ["evaluator", "action", "utils", "test", "types", "constants", "messages", "agents", "relationships", "context", "provider", "logger"] +const ignorePatterns = ["evaluator", "action", "utils", "template", "util", "test", "types", "constants", "agents", "relationships", "context", "provider", "logger"] // __dirname is not defined in ES module scope, so we need to create it const __filename = fileURLToPath(import.meta.url) diff --git a/src/lib/__tests__/memory.test.ts b/src/lib/__tests__/memory.test.ts index a64142e..80f4a3f 100644 --- a/src/lib/__tests__/memory.test.ts +++ b/src/lib/__tests__/memory.test.ts @@ -100,6 +100,17 @@ describe("Memory", () => { }, ); + console.log( + "searchedMemories", + searchedMemories.map((m) => { + return; + { + m.content; + m.room_id; + } + }), + ); + // Check that the similar memory is included in the search results and the dissimilar one is not or ranks lower expect( searchedMemories.some( @@ -249,10 +260,7 @@ describe("Memory - Basic tests", () => { }); // Verify creation by counting memories - const initialCount = await memoryManager.countMemories( - room_id, - false, - ); + const initialCount = await memoryManager.countMemories(room_id, false); expect(initialCount).toBeGreaterThan(0); // Search memories by embedding @@ -267,8 +275,7 @@ describe("Memory - Basic tests", () => { // Remove a specific memory await memoryManager.removeMemory(createdMemories[0].id!); - const afterRemovalCount = - await memoryManager.countMemories(room_id); + const afterRemovalCount = await memoryManager.countMemories(room_id); expect(afterRemovalCount).toBeLessThan(initialCount); // Remove all memories for the test user @@ -437,10 +444,7 @@ describe("Memory - Extended Tests", () => { await memoryManager.createMemory(similarMemory, true); const allCount = await memoryManager.countMemories(room_id, false); - const uniqueCount = await memoryManager.countMemories( - room_id, - true, - ); + const uniqueCount = await memoryManager.countMemories(room_id, true); expect(allCount > uniqueCount).toBe(true); }); diff --git a/src/lib/adapters/sqlite.ts b/src/lib/adapters/sqlite.ts index b6c744e..b2f9d1e 100644 --- a/src/lib/adapters/sqlite.ts +++ b/src/lib/adapters/sqlite.ts @@ -56,6 +56,23 @@ export class SqliteDatabaseAdapter extends DatabaseAdapter { return this.db.prepare(sql).all(params.room_id) as Actor[]; } + async createMemory( + memory: Memory, + tableName: string, + unique = false, + ): Promise { + const sql = `INSERT INTO memories (id, type, content, embedding, user_id, room_id, \`unique\`) VALUES (?, ?, ?, ?, ?, ?, ?)`; + this.db.prepare(sql).run( + crypto.randomUUID(), + tableName, + JSON.stringify(memory.content), // stringify the content field + JSON.stringify(memory.embedding), + memory.user_id, + memory.room_id, + unique ? 1 : 0, + ); + } + async searchMemories(params: { tableName: string; room_id: UUID; @@ -64,19 +81,75 @@ export class SqliteDatabaseAdapter extends DatabaseAdapter { match_count: number; unique: boolean; }): Promise { - let sql = ` - SELECT * + const sql = ` + SELECT *, (1 - vss_distance_l2(embedding, ?)) AS similarity FROM memories WHERE type = ? AND room_id = ? + AND vss_search(embedding, ?) + ORDER BY similarity DESC LIMIT ? `; - const queryParams = [params.tableName, params.room_id, params.match_count]; + const queryParams = [ + JSON.stringify(params.embedding), + params.tableName, + params.room_id, + JSON.stringify(params.embedding), + params.match_count, + ]; if (params.unique) { - sql += " AND `unique` = 1"; + // sql += " AND `unique` = 1"; } - const memories = this.db.prepare(sql).all(...queryParams) as Memory[]; + const memories = this.db.prepare(sql).all(...queryParams) as (Memory & { + similarity: number; + })[]; + return memories.map((memory) => ({ + ...memory, + content: JSON.parse(memory.content as unknown as string), + })); + } + + async searchMemoriesByEmbedding( + embedding: number[], + params: { + match_threshold?: number; + count?: number; + room_id?: UUID; + unique?: boolean; + tableName: string; + }, + ): Promise { + let sql = ` + SELECT *, (1 - vss_distance_l2(embedding, ?)) AS similarity + FROM memories + WHERE type = ? + AND vss_search(embedding, ?) + ORDER BY similarity DESC + `; + const queryParams = [ + JSON.stringify(embedding), + params.tableName, + JSON.stringify(embedding), + ]; + + if (params.room_id) { + // sql += " AND room_id = ?"; + // queryParams.push(params.room_id); + } + + if (params.unique) { + // sql += " AND `unique` = 1"; + } + + if (params.count) { + sql += " LIMIT ?"; + queryParams.push(params.count.toString()); + } + + const memories = this.db.prepare(sql).all(...queryParams) as (Memory & { + similarity: number; + })[]; return memories.map((memory) => ({ ...memory, content: JSON.parse(memory.content as unknown as string), @@ -178,66 +251,6 @@ export class SqliteDatabaseAdapter extends DatabaseAdapter { })); } - async searchMemoriesByEmbedding( - embedding: number[], - params: { - match_threshold?: number; - count?: number; - room_id?: UUID; - unique?: boolean; - tableName: string; - }, - ): Promise { - let sql = ` - SELECT * - FROM memories - WHERE type = ? AND vss_search(embedding, ?) - ORDER BY vss_search(embedding, ?) DESC - `; - const queryParams = [ - params.tableName, - JSON.stringify(embedding), - JSON.stringify(embedding), - ]; - - if (params.room_id) { - sql += " AND room_id = ?"; - queryParams.push(params.room_id); - } - - // if (params.unique) { - // sql += " AND `unique` = 1"; - // } - - if (params.count) { - sql += " LIMIT ?"; - queryParams.push(params.count.toString()); - } - - const memories = this.db.prepare(sql).all(...queryParams) as Memory[]; - return memories.map((memory) => ({ - ...memory, - content: JSON.parse(memory.content as unknown as string), - })); - } - - async createMemory( - memory: Memory, - tableName: string, - unique = false, - ): Promise { - const sql = `INSERT INTO memories (id, type, content, embedding, user_id, room_id, \`unique\`) VALUES (?, ?, ?, ?, ?, ?, ?)`; - this.db.prepare(sql).run( - crypto.randomUUID(), - tableName, - JSON.stringify(memory.content), // stringify the content field - JSON.stringify(memory.embedding), - memory.user_id, - memory.room_id, - unique ? 1 : 0, - ); - } - async removeMemory(memoryId: UUID, tableName: string): Promise { const sql = `DELETE FROM memories WHERE type = ? AND id = ?`; this.db.prepare(sql).run(tableName, memoryId); diff --git a/src/lib/adapters/sqlite/sqliteTables.ts b/src/lib/adapters/sqlite/sqliteTables.ts index 02f3874..4eb1ceb 100644 --- a/src/lib/adapters/sqlite/sqliteTables.ts +++ b/src/lib/adapters/sqlite/sqliteTables.ts @@ -9,7 +9,7 @@ CREATE TABLE IF NOT EXISTS "accounts" ( "name" TEXT, "email" TEXT NOT NULL UNIQUE, "avatar_url" TEXT, - "details" TEXT DEFAULT '{}' + "details" TEXT DEFAULT '{}' CHECK(json_valid("details")) -- Ensuring details is a valid JSON field ); -- Table: memories @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS "memories" ( "type" TEXT NOT NULL, "created_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "content" TEXT NOT NULL, - "embedding" BLOB NOT NULL, + "embedding" BLOB NOT NULL, -- TODO: EMBEDDING ARRAY, CONVERT TO BEST FORMAT FOR SQLITE-VSS (JSON?) "user_id" TEXT, "room_id" TEXT, "unique" INTEGER DEFAULT 1 NOT NULL, @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS "goals" ( "status" TEXT, "description" TEXT, "room_id" TEXT, - "objectives" TEXT DEFAULT '[]' NOT NULL + "objectives" TEXT DEFAULT '[]' NOT NULL CHECK(json_valid("objectives")) -- Ensuring objectives is a valid JSON array ); -- Table: logs