docs - use config.docs instead of inferring from statuses

core/core.ts

@@ -731,9 +731,6 @@ export class Core {
         this.docsService.setPaused(msg.data.id, msg.data.paused);
       }
     });
-    on("indexing/initStatuses", async (msg) => {
-      return this.docsService.initStatuses();
-    });
     on("docs/getSuggestedDocs", async (msg) => {
       if (hasRequestedDocs) {
         return;
@@ -742,6 +739,9 @@ export class Core {
       const suggestedDocs = await getAllSuggestedDocs(this.ide);
       this.messenger.send("docs/suggestions", suggestedDocs);
     });
+    on("docs/initStatuses", async (msg) => {
+      void this.docsService.initStatuses();
+    });
     //
 
     on("didChangeSelectedProfile", (msg) => {

core/index.d.ts

@@ -68,8 +68,6 @@ export interface IndexingStatus {
   url?: string;
 }
 
-export type IndexingStatusMap = Map<string, IndexingStatus>;
-
 export type PromptTemplateFunction = (
   history: ChatMessage[],
   otherData: Record<string, string>,

core/indexing/docs/DocsService.ts

@@ -7,7 +7,6 @@ import {
   ContinueConfig,
   EmbeddingsProvider,
   IDE,
-  IndexingStatusMap,
   IndexingStatus,
   SiteIndexingConfig,
   IdeInfo,
@@ -86,7 +85,6 @@ export type AddParams = {
 export default class DocsService {
   static lanceTableName = "docs";
   static sqlitebTableName = "docs";
-  static indexingType = "docs";
 
   static preIndexedDocsEmbeddingsProvider =
     new TransformersJsEmbeddingsProvider();
@@ -136,38 +134,55 @@ export default class DocsService {
     configHandler.onConfigUpdate(this.handleConfigUpdate.bind(this));
   }
 
-  readonly statuses: IndexingStatusMap = new Map();
-
-  // Function for GUI to retrieve initial pending statuses
-  // And kickoff indexing where needed
-  async initStatuses() {
-    this.config?.docs?.forEach(async (doc) => {
-      const currentStatus = this.statuses.get(doc.startUrl);
-      if (currentStatus) {
-        this.handleStatusUpdate(currentStatus);
-      } else {
-        this.handleStatusUpdate({
-          type: "docs",
-          id: doc.startUrl,
-          embeddingsProviderId: this.config.embeddingsProvider.id,
-          isReindexing: false,
-          progress: 0,
-          description: "Pending",
-          status: "pending",
-          title: doc.title,
-          debugInfo: `max depth: ${doc.maxDepth}`,
-          icon: doc.faviconUrl,
-          url: doc.startUrl,
-        });
-      }
-    });
-  }
+  readonly statuses: Map<string, IndexingStatus> = new Map();
 
   handleStatusUpdate(update: IndexingStatus) {
     this.statuses.set(update.id, update);
     this.messenger?.send("indexing/statusUpdate", update);
   }
 
+  // A way for gui to retrieve initial statuses
+  async initStatuses(): Promise<void> {
+    if (!this.config?.docs) {
+      return;
+    }
+    const metadata = await this.listMetadata();
+
+    this.config.docs?.forEach((doc) => {
+      if (!doc.startUrl) {
+        console.error("Invalid config docs entry", doc);
+        return;
+      }
+
+      const sharedStatus = {
+        type: "docs" as IndexingStatus["type"],
+        id: doc.startUrl,
+        embeddingsProviderId: this.config.embeddingsProvider.id,
+        isReindexing: false,
+        title: doc.title,
+        debugInfo: `max depth: ${doc.maxDepth}`,
+        icon: doc.faviconUrl,
+        url: doc.startUrl,
+      };
+      const indexedStatus: IndexingStatus = metadata.find(
+        (meta) => meta.startUrl === doc.startUrl,
+      )
+        ? {
+            ...sharedStatus,
+            progress: 0,
+            description: "Pending",
+            status: "pending",
+          }
+        : {
+            ...sharedStatus,
+            progress: 1,
+            description: "Complete",
+            status: "complete",
+          };
+      this.handleStatusUpdate(indexedStatus);
+    });
+  }
+
   abort(startUrl: string) {
     const status = this.statuses.get(startUrl);
     if (status) {
@@ -362,6 +377,7 @@ export default class DocsService {
 
     const indexExists = await this.hasMetadata(startUrl);
 
+    // Build status update - most of it is fixed values
     const fixedStatus: Pick<
       IndexingStatus,
       | "type"
@@ -383,6 +399,8 @@ export default class DocsService {
       url: siteIndexingConfig.startUrl,
     };
 
+    // Clear current indexes if reIndexing
+    //
     if (indexExists) {
       if (reIndex) {
         await this.deleteIndexes(startUrl);
@@ -398,6 +416,12 @@ export default class DocsService {
       }
     }
 
+    // If not preindexed
+    const isPreIndexedDoc = !!preIndexedDocs[siteIndexingConfig.startUrl];
+    if (!isPreIndexedDoc) {
+      this.addToConfig(siteIndexingConfig);
+    }
+
     try {
       this.handleStatusUpdate({
         ...fixedStatus,
@@ -435,7 +459,7 @@ export default class DocsService {
         articles.push(article);
 
         processedPages++;
-        await new Promise((resolve) => setTimeout(resolve, 50)); // Locks down GUI if no sleeping
+        await new Promise((resolve) => setTimeout(resolve, 100)); // Locks down GUI if no sleeping
       }
 
       void Telemetry.capture("docs_pages_crawled", {
@@ -788,19 +812,13 @@ export default class DocsService {
         }
       }
 
-      // Sends "Pending" or current status for each
-      await this.initStatuses();
-
-      for (const doc of changedDocs) {
-        console.log(`Updating indexed doc: ${doc.startUrl}`);
-        await this.indexAndAdd(doc, true);
-      }
+      await Promise.allSettled([
+        ...changedDocs.map((doc) => this.indexAndAdd(doc, true)),
+        ...newDocs.map((doc) => this.indexAndAdd(doc)),
+      ]);
 
       for (const doc of newDocs) {
-        console.log(`Indexing new doc: ${doc.startUrl}`);
         void Telemetry.capture("add_docs_config", { url: doc.startUrl });
-
-        await this.indexAndAdd(doc);
       }
 
       for (const doc of deletedDocs) {
@@ -941,7 +959,7 @@ export default class DocsService {
     );
   }
 
-  private addToConfig({ siteIndexingConfig }: AddParams) {
+  private addToConfig(siteIndexingConfig: SiteIndexingConfig) {
     // Handles the case where a user has manually added the doc to config.json
     // so it already exists in the file
     const doesDocExist = this.config.docs?.some(
@@ -959,13 +977,6 @@ export default class DocsService {
   private async add(params: AddParams) {
     await this.addToLance(params);
     await this.addMetadataToSqlite(params);
-
-    const isPreIndexedDoc =
-      !!preIndexedDocs[params.siteIndexingConfig.startUrl];
-
-    if (!isPreIndexedDoc) {
-      this.addToConfig(params);
-    }
   }
 
   // Delete methods
@@ -1030,10 +1041,7 @@ export default class DocsService {
     console.log(
       `Reindexing non-preindexed docs with new embeddings provider: ${embeddingsProvider.id}`,
     );
-    await this.initStatuses();
-    for (const doc of docs) {
-      await this.indexAndAdd(doc, true);
-    }
+    await Promise.allSettled(docs.map((doc) => this.indexAndAdd(doc)));
 
     // Important that this only is invoked after we have successfully
     // cleared and reindex the docs so that the table cannot end up in an

core/protocol/core.ts

@@ -9,6 +9,7 @@ import type {
   DiffLine,
   FileSymbolMap,
   IdeSettings,
+  IndexingStatus,
   LLMFullCompletionOptions,
   MessageContent,
   ModelDescription,
@@ -165,8 +166,8 @@ export type ToCoreFromIdeOrWebviewProtocol = {
   "indexing/reindex": [{ type: string; id: string }, void];
   "indexing/abort": [{ type: string; id: string }, void];
   "indexing/setPaused": [{ type: string; id: string; paused: boolean }, void];
-  "indexing/initStatuses": [undefined, void];
   "docs/getSuggestedDocs": [undefined, void];
+  "docs/initStatuses": [undefined, void];
 
   addAutocompleteModel: [{ model: ModelDescription }, void];
 

core/protocol/passThrough.ts

@@ -48,11 +48,11 @@ export const WEBVIEW_TO_CORE_PASS_THROUGH: (keyof ToCoreFromWebviewProtocol)[] =
     "index/forceReIndexFiles",
     "index/indexingProgressBarInitialized",
     // Docs, etc.
-    "indexing/initStatuses",
     "indexing/reindex",
     "indexing/abort",
     "indexing/setPaused",
     "docs/getSuggestedDocs",
+    "docs/initStatuses",
     //
     "completeOnboarding",
     "addAutocompleteModel",

gui/src/App.tsx

@@ -8,7 +8,7 @@ import useSubmenuContextProviders from "./hooks/useSubmenuContextProviders";
 import { useVscTheme } from "./hooks/useVscTheme";
 import { AddNewModel, ConfigureProvider } from "./pages/AddNewModel";
 import ConfigErrorPage from "./pages/config-error";
-import Edit from "./pages/Edit";
+import Edit from "./pages/edit";
 import ErrorPage from "./pages/error";
 import Chat from "./pages/gui";
 import History from "./pages/history";

gui/src/components/AccountDialog.tsx

@@ -65,6 +65,8 @@ function AccountDialog() {
     );
   }
 
+  const dispatch = useDispatch();
+
   const changeProfileId = (id: string) => {
     ideMessenger.post("didChangeSelectedProfile", { id });
     dispatch(setSelectedProfileId(id));

gui/src/components/dialogs/AddDocsDialog.tsx

@@ -22,7 +22,7 @@ import {
   setShowDialog,
 } from "../../redux/slices/uiStateSlice";
 import { RootState } from "../../redux/store";
-import IndexingStatusViewer from "../indexing/IndexingStatus";
+import IndexingStatusViewer from "../indexing/DocsIndexingStatus";
 
 import { ToolTip } from "../gui/Tooltip";
 import FileIcon from "../FileIcon";

gui/src/components/indexing/DocsIndexingPeeks.tsx

@@ -1,4 +1,4 @@
-import { useDispatch } from "react-redux";
+import { useDispatch, useSelector } from "react-redux";
 
 import { IndexingStatus } from "core";
 import { useMemo } from "react";
@@ -8,6 +8,7 @@ import {
   setDialogMessage,
   setShowDialog,
 } from "../../redux/slices/uiStateSlice";
+import { RootState } from "../../redux/store";
 
 export interface DocsIndexingPeekProps {
   status: IndexingStatus;
@@ -58,6 +59,9 @@ interface DocsIndexingPeeksProps {
 }
 
 function DocsIndexingPeekList({ statuses }: DocsIndexingPeeksProps) {
+  const config = useSelector((store: RootState) => store.state.config);
+  const configDocs = config.docs ?? [];
+  // const docs
   return (
     <div className="border-vsc-input-border mt-2 flex flex-col border-0 border-t border-solid pt-2">
       <p className="mx-0 my-1.5 p-0 px-1 font-semibold text-stone-500">