autodetect docs v1

core/config/ProfileLifecycleManager.ts

@@ -1,5 +1,3 @@
-import { config } from "dotenv";
-
 import {
   BrowserSerializedContinueConfig,
   ContinueConfig,

core/config/load.ts

@@ -527,6 +527,7 @@ function finalToBrowserConfig(
     embeddingsProvider: final.embeddingsProvider?.id,
     ui: final.ui,
     experimental: final.experimental,
+    docs: final.docs,
   };
 }
 

core/core.ts

@@ -735,6 +735,10 @@ export class Core {
       return this.docsService.initStatuses();
     });
     on("docs/getSuggestedDocs", async (msg) => {
+      if (hasRequestedDocs) {
+        return;
+      } // TODO, remove, hack because of rerendering
+      hasRequestedDocs = true;
       const suggestedDocs = await getAllSuggestedDocs(this.ide);
       this.messenger.send("docs/suggestions", suggestedDocs);
     });
@@ -842,3 +846,5 @@ export class Core {
 
   // private
 }
+
+let hasRequestedDocs = false;

core/index.d.ts

@@ -1189,6 +1189,7 @@ export interface BrowserSerializedContinueConfig {
   reranker?: RerankerDescription;
   experimental?: ExperimentalConfig;
   analytics?: AnalyticsConfig;
+  docs?: SiteIndexingConfig[];
 }
 
 // DOCS SUGGESTIONS AND PACKAGE INFO
@@ -1210,6 +1211,7 @@ export type ParsedPackageInfo = {
 
 export type PackageDetails = {
   docsLink?: string;
+  docsLinkWarning?: string;
   title?: string;
   description?: string;
   repo?: string;

core/indexing/docs/suggestions/index.ts

@@ -63,42 +63,40 @@ export async function getAllSuggestedDocs(ide: IDE) {
   );
 
   // Parse package files and build map of language -> packages
-  const packagesByRegistry: Record<string, ParsedPackageInfo[]> = {};
+  const packagesByCrawler: Record<string, ParsedPackageInfo[]> = {};
   PACKAGE_CRAWLERS.forEach((Crawler) => {
     const crawler = new Crawler();
+    packagesByCrawler[crawler.packageRegistry] = [];
     const packageFiles = packageFilesByRegistry[crawler.packageRegistry];
     packageFiles.forEach((file) => {
       const contents = fileContents.get(file.path);
       if (!contents) {
         return;
       }
       const packages = crawler.parsePackageFile(file, contents);
-      if (!packagesByRegistry[crawler.packageRegistry]) {
-        packagesByRegistry[crawler.packageRegistry] = [];
-      }
-      packagesByRegistry[crawler.packageRegistry].push(...packages);
+      packagesByCrawler[crawler.packageRegistry].push(...packages);
     });
   });
 
   // Deduplicate packages per language
   // TODO - this is where you would allow docs for different versions
   // by e.g. using "name-version" as the map key instead of just name
   // For now have not allowed
-  const languages = Object.keys(packagesByRegistry);
-  languages.forEach((language) => {
-    const packages = packagesByRegistry[language];
+  const registries = Object.keys(packagesByCrawler);
+  registries.forEach((registry) => {
+    const packages = packagesByCrawler[registry];
     const uniquePackages = Array.from(
       new Map(packages.map((pkg) => [pkg.name, pkg])).values(),
     );
-    packagesByRegistry[language] = uniquePackages;
+    packagesByCrawler[registry] = uniquePackages;
   });
 
   // Get documentation links for all packages
   const allDocsResults: PackageDocsResult[] = [];
   await Promise.all(
     PACKAGE_CRAWLERS.map(async (Crawler) => {
       const crawler = new Crawler();
-      const packages = packagesByRegistry[crawler.packageRegistry];
+      const packages = packagesByCrawler[crawler.packageRegistry];
       const docsByRegistry = await Promise.all(
         packages.map(async (packageInfo) => {
           try {
@@ -114,6 +112,11 @@ export async function getAllSuggestedDocs(ide: IDE) {
               details: {
                 ...details,
                 docsLink: details.docsLink,
+                docsLinkWarning: details.docsLink.includes("github.com")
+                  ? "Github docs not supported, find the docs site"
+                  : details.docsLink.includes("docs")
+                    ? undefined
+                    : "May not be a docs site, check the URL",
               },
             };
           } catch (error) {
@@ -129,19 +132,3 @@ export async function getAllSuggestedDocs(ide: IDE) {
   );
   return allDocsResults;
 }
-
-// write me an interface PackageCrawler that contains:
-// 1. property `language` to store a given language like "python" or "typescript"
-// 2. has a method `getPackageFiles` which takes a list of file names and decides which ones match package/dependency files (e.g. package.json for typescript, requirements.txt for python, etc)
-// 3. has a method `parsePackageFile` which returns a list of package name and version from a relevant package file, in a standardized format like semver
-// 4. has a method `getDocumentationLink` to check for documentation link for a given package (e.g. GET `https://registry.npmjs.org/<package>` and find docs field for typescript, documentation link in the package metadata for PyPi, etc.)
-// Then, write typescript classes to implement this typescript interface for the languages "python" and "typescript"
-
-// I want to present the user with a list of dependencies and allow them to select which ones to index (embed) documentation for.
-// In order to prevent duplicate file reads, the process will be like this:
-// 1. take in a list of filepaths called `filepaths`
-// 2. loop an array of PackageCrawler classes to build a map of `language` (string) to `packageFilePaths` (string[])
-// 3. Get unique filepaths from `packageFilePaths` and build a map ` of filepath to file contents using an existing `readFile` function, and skipping file reads of already in the map
-// Finally,
-// Add a `` method to the interface and classes that returns
-// Then, assemble the classes in an array, and write a function getAllSuggestedDocs that returns a map of `language` to an ar

core/indexing/docs/suggestions/packageCrawlers/Python.ts

@@ -30,7 +30,7 @@ export class PythonPackageCrawler implements PackageCrawler {
       .split("\n")
       .map((line) => {
         const [name, version] = line.split("==");
-        return { name, version, packageFile: file, language: "python" };
+        return { name, version, packageFile: file, language: "py" };
       })
       .filter((pkg) => pkg.name && pkg.version);
   }

core/protocol/core.ts

@@ -9,7 +9,6 @@ import type {
   DiffLine,
   FileSymbolMap,
   IdeSettings,
-  IndexingStatusMap,
   LLMFullCompletionOptions,
   MessageContent,
   ModelDescription,

core/protocol/webview.ts

@@ -2,9 +2,9 @@ import { ConfigValidationError } from "../config/validation.js";
 
 import type {
   ContextItemWithId,
-  DocsSuggestions,
   IndexingProgressUpdate,
   IndexingStatus,
+  PackageDocsResult,
 } from "../index.js";
 
 export type ToWebviewFromIdeOrCoreProtocol = {
@@ -26,5 +26,5 @@ export type ToWebviewFromIdeOrCoreProtocol = {
   getWebviewHistoryLength: [undefined, number];
   signInToControlPlane: [undefined, void];
   openDialogMessage: ["account", void];
-  "docs/suggestions": [DocsSuggestions, void];
+  "docs/suggestions": [PackageDocsResult[], void];
 };

gui/src/components/dialogs/index.tsx

@@ -71,7 +71,7 @@ const TextDialog = (props: TextDialogProps) => {
   return (
     <ScreenCover onClick={props.onClose} hidden={!props.showDialog}>
       <DialogContainer
-        className="xs:w-[85%] w-[92%] sm:w-[75%]"
+        className="xs:w-[90%] no-scrollbar max-h-full w-[92%] max-w-[600px] overflow-auto sm:w-[88%] md:w-[80%]"
         onClick={(e) => {
           e.stopPropagation();
         }}

gui/src/components/indexing/ChatIndexingPeeks.tsx

@@ -104,7 +104,7 @@ function ChatIndexingPeeks() {
   return (
     <div className="flex flex-col gap-1">
       {mergedIndexingStates.map((state) => {
-        return <ChatIndexingPeek state={state} />;
+        return <ChatIndexingPeek key={state.type} state={state} />;
       })}
     </div>
   );

gui/src/components/indexing/DocsIndexingPeeks.tsx

@@ -0,0 +1,76 @@
+import { useDispatch, useSelector } from "react-redux";
+import { RootState } from "../../redux/store";
+
+import { IndexingStatus } from "core";
+import { useMemo } from "react";
+import { useNavigate } from "react-router-dom";
+import { ArrowPathIcon, EyeSlashIcon } from "@heroicons/react/24/outline";
+import {
+  setDialogMessage,
+  setShowDialog,
+} from "../../redux/slices/uiStateSlice";
+
+export interface DocsIndexingPeekProps {
+  status: IndexingStatus;
+}
+
+function DocsIndexingPeek({ status }: DocsIndexingPeekProps) {
+  const navigate = useNavigate();
+  const dispatch = useDispatch();
+
+  const progressPercentage = useMemo(() => {
+    return Math.min(100, Math.max(0, status.progress * 100));
+  }, [status.progress]);
+
+  return (
+    <div
+      className="flex cursor-pointer flex-row items-center gap-2 rounded-md px-1 text-stone-500 hover:bg-gray-700/10"
+      onClick={() => {
+        navigate("/more");
+        dispatch(setShowDialog(false));
+        dispatch(setDialogMessage(undefined));
+      }}
+    >
+      <p className="m-0 p-0 text-stone-500 group-hover:underline">
+        {status.title}
+      </p>
+      <div className="my-2 h-1.5 flex-1 rounded-md border border-solid border-gray-400">
+        <div
+          className={`h-full rounded-lg bg-stone-500 transition-all duration-200 ease-in-out`}
+          style={{
+            width: `${progressPercentage}%`,
+          }}
+        />
+      </div>
+      <div className="xs:flex hidden flex-row items-center gap-1 text-stone-500">
+        <span className="text-xs no-underline">
+          {progressPercentage.toFixed(0)}%
+        </span>
+        <ArrowPathIcon
+          className={`animate-spin-slow inline-block h-4 w-4 text-stone-500`}
+        ></ArrowPathIcon>
+      </div>
+    </div>
+  );
+}
+
+interface DocsIndexingPeeksProps {
+  statuses: IndexingStatus[];
+}
+
+function DocsIndexingPeekList({ statuses }: DocsIndexingPeeksProps) {
+  if (!statuses.length) return null;
+
+  return (
+    <div className="flex flex-col">
+      <p className="mx-0 my-1.5 p-0 text-stone-500">Currently Indexing</p>
+      <div className="max-h-[100px] overflow-y-auto pr-2">
+        {statuses.map((status) => {
+          return <DocsIndexingPeek key={status.id} status={status} />;
+        })}
+      </div>
+    </div>
+  );
+}
+
+export default DocsIndexingPeekList;

gui/src/components/indexing/IndexingStatuses.tsx

@@ -8,6 +8,8 @@ import {
   setShowDialog,
 } from "../../redux/slices/uiStateSlice";
 import AddDocsDialog from "../dialogs/AddDocsDialog";
+import { PlusCircleIcon } from "@heroicons/react/24/outline";
+import { ToolTip } from "../gui/Tooltip";
 
 function IndexingStatuses() {
   const indexingStatuses = useSelector(
@@ -25,30 +27,35 @@ function IndexingStatuses() {
     <div className="flex flex-col gap-1">
       <div className="flex flex-row items-center justify-between">
         <h3 className="mb-1 mt-0 text-xl">@docs indexes</h3>
-        {/* <div className="border-1 rounded-full border">
-          <ChevronUpIcon className="h-8 w-8" />
-        </div> */}
-        {/* TODO add some way to hide, scroll, etc. */}
+        {docsStatuses.length ? (
+          <>
+            <PlusCircleIcon
+              data-tooltip-id={"more-add-docs-button"}
+              className="text-vsc-foreground-muted h-5 w-5 cursor-pointer focus:ring-0"
+              onClick={() => {
+                dispatch(setShowDialog(true));
+                dispatch(setDialogMessage(<AddDocsDialog />));
+              }}
+            />
+            <ToolTip id={"more-add-docs-button"} place="top">
+              Add Docs
+            </ToolTip>
+          </>
+        ) : null}
       </div>
       <span className="text-xs text-stone-500">
         Manage your documentation sources
       </span>
-      {/* <div className="flex max-h-[170px] flex-col gap-1 overflow-x-hidden overflow-y-scroll pr-2"> */}
-      {docsStatuses.length
-        ? docsStatuses.map((status) => {
+      <div className="flex max-h-[170px] flex-col gap-1 overflow-x-hidden overflow-y-scroll pr-2">
+        {docsStatuses.length ? (
+          docsStatuses.map((status) => {
             return <IndexingStatusViewer key={status.id} status={status} />;
           })
-        : null}
-      <SecondaryButton
-        onClick={() => {
-          dispatch(setShowDialog(true));
-          dispatch(setDialogMessage(<AddDocsDialog />));
-        }}
-      >
-        Add Docs
-      </SecondaryButton>
+        ) : (
+          <SecondaryButton>Add Docs</SecondaryButton>
+        )}
+      </div>
     </div>
-    // </div>
   );
 }