Merge pull request #3623 from continuedev/dallin/docs-embedding-id

Separate Docs EmbeddingProviderId migration
continuedev · Jan 7, 2025 · a4bde28 · a4bde28
2 parents 87d1c19 + c3bd2ca
commit a4bde28
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 10 deletions.
diff --git a/core/indexing/docs/DocsService.ts b/core/indexing/docs/DocsService.ts
@@ -743,10 +743,17 @@ export default class DocsService {
   }
 
   async getFavicon(startUrl: string) {
+    if (!this.config.embeddingsProvider) {
+      console.warn(
+        "Attempting to get favicon without embeddings provider specified",
+      );
+      return;
+    }
     const db = await this.getOrCreateSqliteDb();
     const result = await db.get(
-      `SELECT favicon FROM ${DocsService.sqlitebTableName} WHERE startUrl = ?`,
+      `SELECT favicon FROM ${DocsService.sqlitebTableName} WHERE startUrl = ? AND embeddingsProviderId = ?`,
       startUrl,
+      this.config.embeddingsProvider.embeddingId,
     );
 
     if (!result) {
@@ -1011,10 +1018,18 @@ export default class DocsService {
   }
 
   private async deleteMetadataFromSqlite(startUrl: string) {
+    if (!this.config.embeddingsProvider) {
+      console.warn(
+        `Attempting to delete metadata for ${startUrl} without embeddings provider specified`,
+      );
+      return;
+    }
     const db = await this.getOrCreateSqliteDb();
+
     await db.run(
-      `DELETE FROM ${DocsService.sqlitebTableName} WHERE startUrl = ?`,
+      `DELETE FROM ${DocsService.sqlitebTableName} WHERE startUrl = ? AND embeddingsProviderId = ?`,
       startUrl,
+      this.config.embeddingsProvider.embeddingId,
     );
   }
 

diff --git a/core/indexing/docs/migrations.ts b/core/indexing/docs/migrations.ts
@@ -53,14 +53,6 @@ export async function runSqliteMigrations(db: Database) {
             );
           }
 
-          const hasEmbeddingsProviderColumn = pragma.some(
-            (pragma) => pragma.name === "embeddingsProviderId",
-          );
-          if (!hasEmbeddingsProviderColumn) {
-            // gotta just delete in this case since old docs will be unusable anyway
-            await db.exec(`DROP TABLE ${DocsService.sqlitebTableName};`);
-          }
-
           const needsToUpdateConfig = !hasFaviconCol || hasBaseUrlCol;
           if (needsToUpdateConfig) {
             const sqliteDocs = await db.all<
@@ -78,4 +70,27 @@ export async function runSqliteMigrations(db: Database) {
       () => resolve(undefined),
     );
   });
+
+  await new Promise((resolve) => {
+    void migrate(
+      "sqlite_delete_docs_with_no_embeddingsProviderId",
+      async () => {
+        try {
+          const pragma = await db.all(
+            `PRAGMA table_info(${DocsService.sqlitebTableName});`,
+          );
+          const hasEmbeddingsProviderColumn = pragma.some(
+            (pragma) => pragma.name === "embeddingsProviderId",
+          );
+          if (!hasEmbeddingsProviderColumn) {
+            // gotta just delete in this case since old docs will be unusable anyway
+            await db.exec(`DROP TABLE ${DocsService.sqlitebTableName};`);
+          }
+        } finally {
+          resolve(undefined);
+        }
+      },
+      () => resolve(undefined),
+    );
+  });
 }