Autodetect Docs V1

core/config/ProfileLifecycleManager.ts

@@ -1,5 +1,3 @@
-import { config } from "dotenv";
-
 import {
   BrowserSerializedContinueConfig,
   ContinueConfig,

core/config/load.ts

@@ -527,6 +527,7 @@ function finalToBrowserConfig(
     embeddingsProvider: final.embeddingsProvider?.id,
     ui: final.ui,
     experimental: final.experimental,
+    docs: final.docs,
   };
 }
 

core/core.ts

@@ -19,6 +19,7 @@ import { ControlPlaneClient } from "./control-plane/client";
 import { streamDiffLines } from "./edit/streamDiffLines";
 import { CodebaseIndexer, PauseToken } from "./indexing/CodebaseIndexer";
 import DocsService from "./indexing/docs/DocsService";
+import { getAllSuggestedDocs } from "./indexing/docs/suggestions";
 import { defaultIgnoreFile } from "./indexing/ignore.js";
 import Ollama from "./llm/llms/Ollama";
 import { createNewPromptFileV2 } from "./promptFiles/v2/createNewPromptFile";
@@ -28,11 +29,7 @@ import { DevDataSqliteDb } from "./util/devdataSqlite";
 import { fetchwithRequestOptions } from "./util/fetchWithOptions";
 import { GlobalContext } from "./util/GlobalContext";
 import historyManager from "./util/history";
-import {
-  editConfigJson,
-  getConfigJsonPath,
-  setupInitialDotContinueDirectory,
-} from "./util/paths";
+import { editConfigJson, setupInitialDotContinueDirectory } from "./util/paths";
 import { Telemetry } from "./util/posthog";
 import { getSymbolsForManyFiles } from "./util/treeSitter";
 import { TTS } from "./util/tts";
@@ -734,8 +731,16 @@ export class Core {
         this.docsService.setPaused(msg.data.id, msg.data.paused);
       }
     });
-    on("indexing/initStatuses", async (msg) => {
-      return this.docsService.initStatuses();
+    on("docs/getSuggestedDocs", async (msg) => {
+      if (hasRequestedDocs) {
+        return;
+      } // TODO, remove, hack because of rerendering
+      hasRequestedDocs = true;
+      const suggestedDocs = await getAllSuggestedDocs(this.ide);
+      this.messenger.send("docs/suggestions", suggestedDocs);
+    });
+    on("docs/initStatuses", async (msg) => {
+      void this.docsService.initStatuses();
     });
     //
 
@@ -841,3 +846,5 @@ export class Core {
 
   // private
 }
+
+let hasRequestedDocs = false;

core/index.d.ts

@@ -68,8 +68,6 @@ export interface IndexingStatus {
   url?: string;
 }
 
-export type IndexingStatusMap = Map<string, IndexingStatus>;
-
 export type PromptTemplateFunction = (
   history: ChatMessage[],
   otherData: Record<string, string>,
@@ -1213,4 +1211,42 @@ export interface BrowserSerializedContinueConfig {
   reranker?: RerankerDescription;
   experimental?: ExperimentalConfig;
   analytics?: AnalyticsConfig;
+  docs?: SiteIndexingConfig[];
+}
+
+// DOCS SUGGESTIONS AND PACKAGE INFO
+export interface FilePathAndName {
+  path: string;
+  name: string;
 }
+
+export interface PackageFilePathAndName extends FilePathAndName {
+  packageRegistry: string; // e.g. npm, pypi
+}
+
+export type ParsedPackageInfo = {
+  name: string;
+  packageFile: PackageFilePathAndName;
+  language: string;
+  version: string;
+};
+
+export type PackageDetails = {
+  docsLink?: string;
+  docsLinkWarning?: string;
+  title?: string;
+  description?: string;
+  repo?: string;
+  license?: string;
+};
+
+export type PackageDetailsSuccess = PackageDetails & {
+  docsLink: string;
+};
+
+export type PackageDocsResult = {
+  packageInfo: ParsedPackageInfo;
+} & (
+  | { error: string; details?: never }
+  | { details: PackageDetailsSuccess; error?: never }
+);

core/indexing/docs/DocsService.ts

@@ -7,7 +7,6 @@ import {
   ContinueConfig,
   EmbeddingsProvider,
   IDE,
-  IndexingStatusMap,
   IndexingStatus,
   SiteIndexingConfig,
   IdeInfo,
@@ -87,7 +86,6 @@ export type AddParams = {
 export default class DocsService {
   static lanceTableName = "docs";
   static sqlitebTableName = "docs";
-  static indexingType = "docs";
 
   static preIndexedDocsEmbeddingsProvider =
     new TransformersJsEmbeddingsProvider();
@@ -137,36 +135,58 @@ export default class DocsService {
     configHandler.onConfigUpdate(this.handleConfigUpdate.bind(this));
   }
 
-  readonly statuses: IndexingStatusMap = new Map();
+  readonly statuses: Map<string, IndexingStatus> = new Map();
+
+  handleStatusUpdate(update: IndexingStatus) {
+    this.statuses.set(update.id, update);
+    this.messenger?.send("indexing/statusUpdate", update);
+  }
+
+  // A way for gui to retrieve initial statuses
+  async initStatuses(): Promise<void> {
+    if (!this.config?.docs) {
+      return;
+    }
+    const metadata = await this.listMetadata();
+
+    this.config.docs?.forEach((doc) => {
+      if (!doc.startUrl) {
+        console.error("Invalid config docs entry, no start");
+        return;
+      }
 
-  // Function for GUI to retrieve initial pending statuses
-  // And kickoff indexing where needed
-  async initStatuses() {
-    this.config?.docs?.forEach(async (doc) => {
       const currentStatus = this.statuses.get(doc.startUrl);
       if (currentStatus) {
-        this.handleStatusUpdate(currentStatus);
-      } else {
-        this.handleStatusUpdate({
-          type: "docs",
-          id: doc.startUrl,
-          embeddingsProviderId: this.config.embeddingsProvider.id,
-          isReindexing: false,
-          progress: 0,
-          description: "Pending",
-          status: "pending",
-          title: doc.title,
-          debugInfo: `max depth: ${doc.maxDepth}`,
-          icon: doc.faviconUrl,
-          url: doc.startUrl,
-        });
+        return currentStatus;
       }
-    });
-  }
 
-  handleStatusUpdate(update: IndexingStatus) {
-    this.statuses.set(update.id, update);
-    this.messenger?.send("indexing/statusUpdate", update);
+      const sharedStatus = {
+        type: "docs" as IndexingStatus["type"],
+        id: doc.startUrl,
+        embeddingsProviderId: this.config.embeddingsProvider.id,
+        isReindexing: false,
+        title: doc.title,
+        debugInfo: `max depth: ${doc.maxDepth}`,
+        icon: doc.faviconUrl,
+        url: doc.startUrl,
+      };
+      const indexedStatus: IndexingStatus = metadata.find(
+        (meta) => meta.startUrl === doc.startUrl,
+      )
+        ? {
+            ...sharedStatus,
+            progress: 0,
+            description: "Pending",
+            status: "pending",
+          }
+        : {
+            ...sharedStatus,
+            progress: 1,
+            description: "Complete",
+            status: "complete",
+          };
+      this.handleStatusUpdate(indexedStatus);
+    });
   }
 
   abort(startUrl: string) {
@@ -363,6 +383,7 @@ export default class DocsService {
 
     const indexExists = await this.hasMetadata(startUrl);
 
+    // Build status update - most of it is fixed values
     const fixedStatus: Pick<
       IndexingStatus,
       | "type"
@@ -384,6 +405,8 @@ export default class DocsService {
       url: siteIndexingConfig.startUrl,
     };
 
+    // Clear current indexes if reIndexing
+    //
     if (indexExists) {
       if (reIndex) {
         await this.deleteIndexes(startUrl);
@@ -399,6 +422,12 @@ export default class DocsService {
       }
     }
 
+    // If not preindexed
+    const isPreIndexedDoc = !!preIndexedDocs[siteIndexingConfig.startUrl];
+    if (!isPreIndexedDoc) {
+      this.addToConfig(siteIndexingConfig);
+    }
+
     try {
       this.handleStatusUpdate({
         ...fixedStatus,
@@ -436,7 +465,7 @@ export default class DocsService {
         articles.push(article);
 
         processedPages++;
-        await new Promise((resolve) => setTimeout(resolve, 50)); // Locks down GUI if no sleeping
+        await new Promise((resolve) => setTimeout(resolve, 100)); // Locks down GUI if no sleeping
       }
 
       void Telemetry.capture("docs_pages_crawled", {
@@ -789,19 +818,13 @@ export default class DocsService {
         }
       }
 
-      // Sends "Pending" or current status for each
-      await this.initStatuses();
-
-      for (const doc of changedDocs) {
-        console.log(`Updating indexed doc: ${doc.startUrl}`);
-        await this.indexAndAdd(doc, true);
-      }
+      await Promise.allSettled([
+        ...changedDocs.map((doc) => this.indexAndAdd(doc, true)),
+        ...newDocs.map((doc) => this.indexAndAdd(doc)),
+      ]);
 
       for (const doc of newDocs) {
-        console.log(`Indexing new doc: ${doc.startUrl}`);
         void Telemetry.capture("add_docs_config", { url: doc.startUrl });
-
-        await this.indexAndAdd(doc);
       }
 
       for (const doc of deletedDocs) {
@@ -942,7 +965,7 @@ export default class DocsService {
     );
   }
 
-  private addToConfig({ siteIndexingConfig }: AddParams) {
+  private addToConfig(siteIndexingConfig: SiteIndexingConfig) {
     // Handles the case where a user has manually added the doc to config.json
     // so it already exists in the file
     const doesDocExist = this.config.docs?.some(
@@ -960,13 +983,6 @@ export default class DocsService {
   private async add(params: AddParams) {
     await this.addToLance(params);
     await this.addMetadataToSqlite(params);
-
-    const isPreIndexedDoc =
-      !!preIndexedDocs[params.siteIndexingConfig.startUrl];
-
-    if (!isPreIndexedDoc) {
-      this.addToConfig(params);
-    }
   }
 
   // Delete methods
@@ -1031,10 +1047,7 @@ export default class DocsService {
     console.log(
       `Reindexing non-preindexed docs with new embeddings provider: ${embeddingsProvider.id}`,
     );
-    await this.initStatuses();
-    for (const doc of docs) {
-      await this.indexAndAdd(doc, true);
-    }
+    await Promise.allSettled(docs.map((doc) => this.indexAndAdd(doc)));
 
     // Important that this only is invoked after we have successfully
     // cleared and reindex the docs so that the table cannot end up in an