Still vss isuess

JoinTheAlliance · Mar 19, 2024 · 94e82b4 · 94e82b4
1 parent 8959759
commit 94e82b4
Show file tree

Hide file tree

Showing 4 changed files with 96 additions and 79 deletions.
diff --git a/scripts/concat.mjs b/scripts/concat.mjs
@@ -5,7 +5,7 @@ import { fileURLToPath } from 'url'
 const instructions = 'The above code was taken from my codebase at https://github.com/jointhealliance/bgent.'
 
 // Patterns to ignore
-const ignorePatterns = ["evaluator", "action", "utils", "test", "types", "constants", "messages", "agents", "relationships", "context", "provider", "logger"]
+const ignorePatterns = ["evaluator", "action", "utils", "template", "util", "test", "types", "constants", "agents", "relationships", "context", "provider", "logger"]
 
 // __dirname is not defined in ES module scope, so we need to create it
 const __filename = fileURLToPath(import.meta.url)

diff --git a/src/lib/__tests__/memory.test.ts b/src/lib/__tests__/memory.test.ts
@@ -100,6 +100,17 @@ describe("Memory", () => {
       },
     );
 
+    console.log(
+      "searchedMemories",
+      searchedMemories.map((m) => {
+        return;
+        {
+          m.content;
+          m.room_id;
+        }
+      }),
+    );
+
     // Check that the similar memory is included in the search results and the dissimilar one is not or ranks lower
     expect(
       searchedMemories.some(
@@ -249,10 +260,7 @@ describe("Memory - Basic tests", () => {
     });
 
     // Verify creation by counting memories
-    const initialCount = await memoryManager.countMemories(
-      room_id,
-      false,
-    );
+    const initialCount = await memoryManager.countMemories(room_id, false);
     expect(initialCount).toBeGreaterThan(0);
 
     // Search memories by embedding
@@ -267,8 +275,7 @@ describe("Memory - Basic tests", () => {
 
     // Remove a specific memory
     await memoryManager.removeMemory(createdMemories[0].id!);
-    const afterRemovalCount =
-      await memoryManager.countMemories(room_id);
+    const afterRemovalCount = await memoryManager.countMemories(room_id);
     expect(afterRemovalCount).toBeLessThan(initialCount);
 
     // Remove all memories for the test user
@@ -437,10 +444,7 @@ describe("Memory - Extended Tests", () => {
     await memoryManager.createMemory(similarMemory, true);
 
     const allCount = await memoryManager.countMemories(room_id, false);
-    const uniqueCount = await memoryManager.countMemories(
-      room_id,
-      true,
-    );
+    const uniqueCount = await memoryManager.countMemories(room_id, true);
 
     expect(allCount > uniqueCount).toBe(true);
   });

diff --git a/src/lib/adapters/sqlite.ts b/src/lib/adapters/sqlite.ts
@@ -56,6 +56,23 @@ export class SqliteDatabaseAdapter extends DatabaseAdapter {
     return this.db.prepare(sql).all(params.room_id) as Actor[];
   }
 
+  async createMemory(
+    memory: Memory,
+    tableName: string,
+    unique = false,
+  ): Promise<void> {
+    const sql = `INSERT INTO memories (id, type, content, embedding, user_id, room_id, \`unique\`) VALUES (?, ?, ?, ?, ?, ?, ?)`;
+    this.db.prepare(sql).run(
+      crypto.randomUUID(),
+      tableName,
+      JSON.stringify(memory.content), // stringify the content field
+      JSON.stringify(memory.embedding),
+      memory.user_id,
+      memory.room_id,
+      unique ? 1 : 0,
+    );
+  }
+
   async searchMemories(params: {
     tableName: string;
     room_id: UUID;
@@ -64,19 +81,75 @@ export class SqliteDatabaseAdapter extends DatabaseAdapter {
     match_count: number;
     unique: boolean;
   }): Promise<Memory[]> {
-    let sql = `
-      SELECT *
+    const sql = `
+      SELECT *, (1 - vss_distance_l2(embedding, ?)) AS similarity
       FROM memories
       WHERE type = ? AND room_id = ?
+      AND vss_search(embedding, ?)
+      ORDER BY similarity DESC
       LIMIT ?
     `;
-    const queryParams = [params.tableName, params.room_id, params.match_count];
+    const queryParams = [
+      JSON.stringify(params.embedding),
+      params.tableName,
+      params.room_id,
+      JSON.stringify(params.embedding),
+      params.match_count,
+    ];
 
     if (params.unique) {
-      sql += " AND `unique` = 1";
+      // sql += " AND `unique` = 1";
     }
 
-    const memories = this.db.prepare(sql).all(...queryParams) as Memory[];
+    const memories = this.db.prepare(sql).all(...queryParams) as (Memory & {
+      similarity: number;
+    })[];
+    return memories.map((memory) => ({
+      ...memory,
+      content: JSON.parse(memory.content as unknown as string),
+    }));
+  }
+
+  async searchMemoriesByEmbedding(
+    embedding: number[],
+    params: {
+      match_threshold?: number;
+      count?: number;
+      room_id?: UUID;
+      unique?: boolean;
+      tableName: string;
+    },
+  ): Promise<Memory[]> {
+    let sql = `
+      SELECT *, (1 - vss_distance_l2(embedding, ?)) AS similarity
+      FROM memories
+      WHERE type = ?
+      AND vss_search(embedding, ?)
+      ORDER BY similarity DESC
+    `;
+    const queryParams = [
+      JSON.stringify(embedding),
+      params.tableName,
+      JSON.stringify(embedding),
+    ];
+
+    if (params.room_id) {
+      // sql += " AND room_id = ?";
+      // queryParams.push(params.room_id);
+    }
+
+    if (params.unique) {
+      // sql += " AND `unique` = 1";
+    }
+
+    if (params.count) {
+      sql += " LIMIT ?";
+      queryParams.push(params.count.toString());
+    }
+
+    const memories = this.db.prepare(sql).all(...queryParams) as (Memory & {
+      similarity: number;
+    })[];
     return memories.map((memory) => ({
       ...memory,
       content: JSON.parse(memory.content as unknown as string),
@@ -178,66 +251,6 @@ export class SqliteDatabaseAdapter extends DatabaseAdapter {
     }));
   }
 
-  async searchMemoriesByEmbedding(
-    embedding: number[],
-    params: {
-      match_threshold?: number;
-      count?: number;
-      room_id?: UUID;
-      unique?: boolean;
-      tableName: string;
-    },
-  ): Promise<Memory[]> {
-    let sql = `
-      SELECT *
-      FROM memories
-      WHERE type = ? AND vss_search(embedding, ?)
-      ORDER BY vss_search(embedding, ?) DESC
-    `;
-    const queryParams = [
-      params.tableName,
-      JSON.stringify(embedding),
-      JSON.stringify(embedding),
-    ];
-
-    if (params.room_id) {
-      sql += " AND room_id = ?";
-      queryParams.push(params.room_id);
-    }
-
-    // if (params.unique) {
-    //   sql += " AND `unique` = 1";
-    // }
-
-    if (params.count) {
-      sql += " LIMIT ?";
-      queryParams.push(params.count.toString());
-    }
-
-    const memories = this.db.prepare(sql).all(...queryParams) as Memory[];
-    return memories.map((memory) => ({
-      ...memory,
-      content: JSON.parse(memory.content as unknown as string),
-    }));
-  }
-
-  async createMemory(
-    memory: Memory,
-    tableName: string,
-    unique = false,
-  ): Promise<void> {
-    const sql = `INSERT INTO memories (id, type, content, embedding, user_id, room_id, \`unique\`) VALUES (?, ?, ?, ?, ?, ?, ?)`;
-    this.db.prepare(sql).run(
-      crypto.randomUUID(),
-      tableName,
-      JSON.stringify(memory.content), // stringify the content field
-      JSON.stringify(memory.embedding),
-      memory.user_id,
-      memory.room_id,
-      unique ? 1 : 0,
-    );
-  }
-
   async removeMemory(memoryId: UUID, tableName: string): Promise<void> {
     const sql = `DELETE FROM memories WHERE type = ? AND id = ?`;
     this.db.prepare(sql).run(tableName, memoryId);

diff --git a/src/lib/adapters/sqlite/sqliteTables.ts b/src/lib/adapters/sqlite/sqliteTables.ts
@@ -9,7 +9,7 @@ CREATE TABLE IF NOT EXISTS "accounts" (
     "name" TEXT,
     "email" TEXT NOT NULL UNIQUE,
     "avatar_url" TEXT,
-    "details" TEXT DEFAULT '{}'
+    "details" TEXT DEFAULT '{}' CHECK(json_valid("details")) -- Ensuring details is a valid JSON field
 );
 
 -- Table: memories
@@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS "memories" (
     "type" TEXT NOT NULL,
     "created_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     "content" TEXT NOT NULL,
-    "embedding" BLOB NOT NULL,
+    "embedding" BLOB NOT NULL, -- TODO: EMBEDDING ARRAY, CONVERT TO BEST FORMAT FOR SQLITE-VSS (JSON?)
     "user_id" TEXT,
     "room_id" TEXT,
     "unique" INTEGER DEFAULT 1 NOT NULL,
@@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS "goals" (
     "status" TEXT,
     "description" TEXT,
     "room_id" TEXT,
-    "objectives" TEXT DEFAULT '[]' NOT NULL
+    "objectives" TEXT DEFAULT '[]' NOT NULL CHECK(json_valid("objectives")) -- Ensuring objectives is a valid JSON array
 );
 
 -- Table: logs