From cb071a74194f656a462f53ed503ba725c31958e3 Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Thu, 21 Nov 2024 00:42:09 -0800
Subject: [PATCH 01/17] package file parsers

---
 core/core.ts                                  |   4 +
 core/indexing/docs/suggestions/index.ts       | 100 +++++++++++++++++
 .../suggestions/packageCrawlers/Python.ts     |  33 ++++++
 .../docs/suggestions/packageCrawlers/TsJs.ts  |  33 ++++++
 core/protocol/core.ts                         |   1 +
 core/protocol/passThrough.ts                  |   1 +
 extensions/vscode/package-lock.json           |   4 +-
 gui/src/hooks/useSetup.ts                     |   3 +-
 manual-testing-sandbox/package.json           | 102 ++++++++++++++++++
 manual-testing-sandbox/requirements.txt       |  10 ++
 10 files changed, 287 insertions(+), 4 deletions(-)
 create mode 100644 core/indexing/docs/suggestions/index.ts
 create mode 100644 core/indexing/docs/suggestions/packageCrawlers/Python.ts
 create mode 100644 core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
 create mode 100644 manual-testing-sandbox/package.json
 create mode 100644 manual-testing-sandbox/requirements.txt

diff --git a/core/core.ts b/core/core.ts
index c6b336c7d0..d080ec34b0 100644
--- a/core/core.ts
+++ b/core/core.ts
@@ -40,6 +40,7 @@ import { TTS } from "./util/tts";
 import type { ContextItemId, IDE, IndexingProgressUpdate } from ".";
 import type { FromCoreProtocol, ToCoreProtocol } from "./protocol";
 import type { IMessenger, Message } from "./util/messenger";
+import { getAllSuggestedDocs } from "./indexing/docs/suggestions";
 
 export class Core {
   // implements IMessenger<ToCoreProtocol, FromCoreProtocol>
@@ -737,6 +738,9 @@ export class Core {
     on("indexing/initStatuses", async (msg) => {
       return this.docsService.initStatuses();
     });
+    on("docs/getSuggestedDocs", async (msg) => {
+      getAllSuggestedDocs(this.ide);
+    });
     //
 
     on("didChangeSelectedProfile", (msg) => {
diff --git a/core/indexing/docs/suggestions/index.ts b/core/indexing/docs/suggestions/index.ts
new file mode 100644
index 0000000000..d3d68aca29
--- /dev/null
+++ b/core/indexing/docs/suggestions/index.ts
@@ -0,0 +1,100 @@
+// write me an interface PackageCrawler that contains:
+// 1. property `language` to store a given language like "python" or "typescript"
+// 2. has a method `getPackageFiles` which takes a list of file names and decides which ones match package/dependency files (e.g. package.json for typescript, requirements.txt for python, etc)
+// 3. has a method `parsePackageFile` which returns a list of package name and version from a relevant package file, in a standardized format like semver
+// 4. has a method `getDocumentationLink` to check for documentation link for a given package (e.g. GET `https://registry.npmjs.org/<package>` and find docs field for typescript, documentation link in the package metadata for PyPi, etc.)
+// Then, write typescript classes to implement this typescript interface for the languages "python" and "typescript"
+
+import { IDE } from "../../..";
+import { walkDir } from "../../walkDir";
+import { PythonPackageCrawler } from "./packageCrawlers/Python";
+import { TypeScriptPackageCrawler } from "./packageCrawlers/TsJs";
+
+const PACKAGE_CRAWLERS = [TypeScriptPackageCrawler, PythonPackageCrawler];
+
+export interface PackageCrawler {
+  language: string;
+  getPackageFiles(fileNames: string[]): string[];
+  parsePackageFile(fileContent: string, filePath: string): PackageInfo[];
+  getDocumentationLink(packageName: string): Promise<PackageDocsResult>;
+}
+
+export type PackageInfo = {
+  name: string;
+  version: string;
+  foundInFilepath: string;
+};
+export type PackageDocsResult = PackageInfo &
+  ({ error: string; link?: never } | { link: string; error?: never });
+
+export async function getAllSuggestedDocs(ide: IDE) {
+  const workspaceDirs = await ide.getWorkspaceDirs();
+  const results = await Promise.all(
+    workspaceDirs.map((dir) => {
+      return walkDir(dir, ide);
+    }),
+  );
+  const allPaths = results.flat(); // TODO only get files, not dirs. Not critical for now
+  const allFiles = allPaths.map((path) => path.split("/").pop()!);
+  const packageFilesByLanguage: Record<string, string[]> = {};
+  for (const Crawler of PACKAGE_CRAWLERS) {
+    const crawler = new Crawler();
+    const packageFilePaths = crawler.getPackageFiles(allFiles);
+    packageFilesByLanguage[crawler.language] = packageFilePaths;
+  }
+
+  const uniqueFilePaths = Array.from(
+    new Set(Object.values(packageFilesByLanguage).flat()),
+  );
+  const fileContentsArray = await Promise.all(
+    uniqueFilePaths.map(async (path) => {
+      const contents = await ide.readFile(path);
+      return { path, contents };
+    }),
+  );
+  const fileContents = new Map(
+    fileContentsArray.map(({ path, contents }) => [path, contents]),
+  );
+
+  const packagesByLanguage: Record<string, PackageInfo[]> = {};
+  PACKAGE_CRAWLERS.forEach((Crawler) => {
+    const crawler = new Crawler();
+    const packageFiles = packageFilesByLanguage[crawler.language];
+    packageFiles.forEach((file) => {
+      const contents = fileContents.get(file);
+      if (!contents) {
+        return;
+      }
+      const packages = crawler.parsePackageFile(contents, file);
+      if (!packagesByLanguage[crawler.language]) {
+        packagesByLanguage[crawler.language] = [];
+      }
+      packagesByLanguage[crawler.language].push(...packages);
+    });
+  });
+  console.log(packagesByLanguage);
+  //   const allPackages = await Promise.all(
+  //     PACKAGE_CRAWLERS.map(async (Crawler) => {
+  //         const crawler = new Crawler();
+  //         const packageInfos = uniqueFileContents
+  //             .filter(({ path }) => languageToFilePaths[crawler.language].includes(path))
+  //             .map(({ path, contents }) => crawler.parsePackageFile(contents, path))
+  //             .flat();
+  //         return packageInfos;
+  //     })
+  //   );
+  //   for (const Crawler of PACKAGE_CRAWLERS) {
+  //     const crawler = new Crawler();
+  //     const packages = crawler.parsePackageFile()
+  //     languageToFilePaths[crawler.language] = packageFilePaths;
+  //   }
+}
+
+// I want to present the user with a list of dependencies and allow them to select which ones to index (embed) documentation for.
+// In order to prevent duplicate file reads, the process will be like this:
+// 1. take in a list of filepaths called `filepaths`
+// 2. loop an array of PackageCrawler classes to build a map of `language` (string) to `packageFilePaths` (string[])
+// 3. Get unique filepaths from `packageFilePaths` and build a map ` of filepath to file contents using an existing `readFile` function, and skipping file reads of already in the map
+// Finally,
+// Add a `` method to the interface and classes that returns
+// Then, assemble the classes in an array, and write a function getAllSuggestedDocs that returns a map of `language` to an ar
diff --git a/core/indexing/docs/suggestions/packageCrawlers/Python.ts b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
new file mode 100644
index 0000000000..ac06997663
--- /dev/null
+++ b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
@@ -0,0 +1,33 @@
+import { PackageCrawler, PackageDocsResult, PackageInfo } from "..";
+
+export class PythonPackageCrawler implements PackageCrawler {
+  language = "python";
+
+  getPackageFiles(fileNames: string[]): string[] {
+    // For Python, we typically look for files like requirements.txt or Pipfile
+    return fileNames.filter(
+      (fileName) => fileName === "requirements.txt" || fileName === "Pipfile",
+    );
+  }
+
+  parsePackageFile(fileContent: string, filepath: string): PackageInfo[] {
+    // Assume the fileContent is a string from a requirements.txt formatted file
+    return fileContent
+      .split("\n")
+      .map((line) => {
+        const [name, version] = line.split("==");
+        return { name, version, foundInFilepath: filepath };
+      })
+      .filter((pkg) => pkg.name && pkg.version);
+  }
+
+  async getDocumentationLink(packageName: string): Promise<PackageDocsResult> {
+    // Fetch metadata from PyPI to find the documentation link
+    const response = await fetch(`https://pypi.org/pypi/${packageName}/json`);
+    if (!response.ok) {
+      throw new Error(`Could not fetch data for package ${packageName}`);
+    }
+    const data = await response.json();
+    return data.info.project_urls?.Documentation;
+  }
+}
diff --git a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
new file mode 100644
index 0000000000..fb073e2a1d
--- /dev/null
+++ b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
@@ -0,0 +1,33 @@
+import { PackageCrawler, PackageDocsResult, PackageInfo } from "..";
+
+export class TypeScriptPackageCrawler implements PackageCrawler {
+  language = "typescript";
+
+  getPackageFiles(fileNames: string[]): string[] {
+    // For TypeScript, we look for package.json file
+    return fileNames.filter((fileName) => fileName === "package.json");
+  }
+
+  parsePackageFile(fileContent: string, filepath: string): PackageInfo[] {
+    // Parse the package.json content
+    const jsonData = JSON.parse(fileContent) as Record<string, Object>;
+    const dependencies = Object.entries(jsonData.dependencies || {}).concat(
+      Object.entries(jsonData.devDependencies || {}),
+    );
+    return dependencies.map(([name, version]) => ({
+      name,
+      version,
+      foundInFilepath: filepath,
+    }));
+  }
+
+  async getDocumentationLink(packageName: string): Promise<PackageDocsResult> {
+    // Fetch metadata from the NPM registry to find the documentation link
+    const response = await fetch(`https://registry.npmjs.org/${packageName}`);
+    if (!response.ok) {
+      throw new Error(`Could not fetch data for package ${packageName}`);
+    }
+    const data = await response.json();
+    return data.homepage;
+  }
+}
diff --git a/core/protocol/core.ts b/core/protocol/core.ts
index c757e5d347..069f9e297a 100644
--- a/core/protocol/core.ts
+++ b/core/protocol/core.ts
@@ -167,6 +167,7 @@ export type ToCoreFromIdeOrWebviewProtocol = {
   "indexing/abort": [{ type: string; id: string }, void];
   "indexing/setPaused": [{ type: string; id: string; paused: boolean }, void];
   "indexing/initStatuses": [undefined, void];
+  "docs/getSuggestedDocs": [undefined, void];
 
   addAutocompleteModel: [{ model: ModelDescription }, void];
 
diff --git a/core/protocol/passThrough.ts b/core/protocol/passThrough.ts
index 40d537d41b..ebe2a86977 100644
--- a/core/protocol/passThrough.ts
+++ b/core/protocol/passThrough.ts
@@ -52,6 +52,7 @@ export const WEBVIEW_TO_CORE_PASS_THROUGH: (keyof ToCoreFromWebviewProtocol)[] =
     "indexing/reindex",
     "indexing/abort",
     "indexing/setPaused",
+    "docs/getSuggestedDocs",
     //
     "completeOnboarding",
     "addAutocompleteModel",
diff --git a/extensions/vscode/package-lock.json b/extensions/vscode/package-lock.json
index 9405035051..b30a951dfc 100644
--- a/extensions/vscode/package-lock.json
+++ b/extensions/vscode/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "continue",
-  "version": "0.9.232",
+  "version": "0.9.233",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "continue",
-      "version": "0.9.232",
+      "version": "0.9.233",
       "license": "Apache-2.0",
       "dependencies": {
         "@electron/rebuild": "^3.2.10",
diff --git a/gui/src/hooks/useSetup.ts b/gui/src/hooks/useSetup.ts
index 4bef9da9ce..f183798890 100644
--- a/gui/src/hooks/useSetup.ts
+++ b/gui/src/hooks/useSetup.ts
@@ -84,6 +84,7 @@ function useSetup(dispatch: Dispatch) {
 
   // ON LOAD
   useEffect(() => {
+    ideMessenger.post("docs/getSuggestedDocs", undefined);
     // Override persisted state
     dispatch(setInactive());
 
@@ -178,8 +179,6 @@ function useSetup(dispatch: Dispatch) {
     },
     [defaultModelTitle],
   );
-
-
 }
 
 export default useSetup;
diff --git a/manual-testing-sandbox/package.json b/manual-testing-sandbox/package.json
new file mode 100644
index 0000000000..ef77ac910c
--- /dev/null
+++ b/manual-testing-sandbox/package.json
@@ -0,0 +1,102 @@
+{
+  "name": "gui",
+  "private": true,
+  "type": "module",
+  "author": "Continue Dev, Inc",
+  "license": "Apache-2.0",
+  "scripts": {
+    "dev": "vite",
+    "tsc:check": "tsc -p ./ --noEmit",
+    "build": "tsc && vite build",
+    "preview": "vite preview",
+    "test": "vitest run",
+    "test:coverage": "vitest run --coverage",
+    "test:ui": "vitest --ui",
+    "test:watch": "vitest"
+  },
+  "dependencies": {
+    "@headlessui/react": "^1.7.17",
+    "@heroicons/react": "^2.0.18",
+    "@monaco-editor/react": "^4.6.0",
+    "@reduxjs/toolkit": "^1.9.3",
+    "@tiptap/core": "^2.3.2",
+    "@tiptap/extension-document": "^2.3.2",
+    "@tiptap/extension-dropcursor": "^2.1.16",
+    "@tiptap/extension-history": "^2.3.2",
+    "@tiptap/extension-image": "^2.1.16",
+    "@tiptap/extension-mention": "^2.1.13",
+    "@tiptap/extension-paragraph": "^2.3.2",
+    "@tiptap/extension-placeholder": "^2.1.13",
+    "@tiptap/extension-text": "^2.3.2",
+    "@tiptap/pm": "^2.1.13",
+    "@tiptap/react": "^2.1.13",
+    "@tiptap/starter-kit": "^2.1.13",
+    "@tiptap/suggestion": "^2.1.13",
+    "@types/vscode-webview": "^1.57.1",
+    "core": "file:../core",
+    "dompurify": "^3.0.6",
+    "downshift": "^7.6.0",
+    "lodash": "^4.17.21",
+    "minisearch": "^7.0.2",
+    "onigasm": "^2.2.5",
+    "posthog-js": "^1.130.1",
+    "prismjs": "^1.29.0",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "react-error-boundary": "^4.0.11",
+    "react-hook-form": "^7.47.0",
+    "react-intersection-observer": "^9.13.1",
+    "react-markdown": "^9.0.1",
+    "react-redux": "^8.0.5",
+    "react-remark": "^2.1.0",
+    "react-router-dom": "^6.14.2",
+    "react-switch": "^7.0.0",
+    "react-syntax-highlighter": "^15.5.0",
+    "react-tooltip": "^5.18.0",
+    "redux-persist": "^6.0.0",
+    "redux-persist-transform-filter": "^0.0.22",
+    "rehype-highlight": "^7.0.0",
+    "rehype-katex": "^7.0.0",
+    "rehype-wrap-all": "^1.1.0",
+    "remark-math": "^6.0.0",
+    "reselect": "^5.1.1",
+    "seti-file-icons": "^0.0.8",
+    "socket.io-client": "^4.7.2",
+    "styled-components": "^5.3.6",
+    "table": "^6.8.1",
+    "tippy.js": "^6.3.7",
+    "unist-util-visit": "^5.0.0",
+    "uuid": "^9.0.1",
+    "vscode-webview": "^1.0.1-beta.1"
+  },
+  "devDependencies": {
+    "@swc/cli": "^0.3.14",
+    "@swc/core": "^1.7.26",
+    "@testing-library/dom": "^10.4.0",
+    "@testing-library/jest-dom": "^6.5.0",
+    "@testing-library/react": "^16.0.1",
+    "@testing-library/user-event": "^14.5.2",
+    "@types/lodash": "^4.17.6",
+    "@types/node": "^20.5.6",
+    "@types/node-fetch": "^2.6.4",
+    "@types/react": "^18.3.11",
+    "@types/react-dom": "^18.3.1",
+    "@types/react-router-dom": "^5.3.3",
+    "@types/react-syntax-highlighter": "^15.5.7",
+    "@types/styled-components": "^5.1.26",
+    "@vitejs/plugin-react-swc": "^3.7.0",
+    "@vitest/coverage-v8": "^2.1.3",
+    "@vitest/ui": "^2.1.3",
+    "autoprefixer": "^10.4.13",
+    "jsdom": "^25.0.1",
+    "postcss": "^8.4.21",
+    "tailwindcss": "^3.2.7",
+    "typescript": "^4.9.3",
+    "vite": "^4.1.0",
+    "vitest": "^2.1.3"
+  },
+  "engine-strict": true,
+  "engines": {
+    "node": ">=20.11.0"
+  }
+}
diff --git a/manual-testing-sandbox/requirements.txt b/manual-testing-sandbox/requirements.txt
new file mode 100644
index 0000000000..793d6a8275
--- /dev/null
+++ b/manual-testing-sandbox/requirements.txt
@@ -0,0 +1,10 @@
+flask==2.1.1
+requests==2.28.1
+numpy==1.23.4
+pandas==1.5.0
+scipy==1.9.3
+django==4.1.3
+matplotlib==3.6.2
+pytest==7.2.0
+sqlalchemy==1.4.41
+beautifulsoup4==4.11.1
\ No newline at end of file

From 954bd5f563a278c04f40f0ec40bfbe27fc497afb Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Thu, 21 Nov 2024 11:58:48 -0800
Subject: [PATCH 02/17] python and typescript crawlers

---
 core/index.d.ts                               |  29 +++++
 core/indexing/docs/suggestions/index.ts       | 115 +++++++++++-------
 .../suggestions/packageCrawlers/Python.ts     |  39 ++++--
 .../docs/suggestions/packageCrawlers/TsJs.ts  |  40 ++++--
 4 files changed, 159 insertions(+), 64 deletions(-)

diff --git a/core/index.d.ts b/core/index.d.ts
index 0646edc541..de1adb8934 100644
--- a/core/index.d.ts
+++ b/core/index.d.ts
@@ -1190,3 +1190,32 @@ export interface BrowserSerializedContinueConfig {
   experimental?: ExperimentalConfig;
   analytics?: AnalyticsConfig;
 }
+
+// DOCS SUGGESTIONS AND PACKAGE INFO
+export interface FilePathAndName {
+  path: string;
+  name: string;
+}
+export type ParsedPackageInfo = {
+  language: string;
+  name: string;
+  version: string;
+  packageFile: FilePathAndName;
+};
+
+export type PackageDetails = {
+  docsLink?: string;
+  title?: string;
+  description?: string;
+};
+
+export type PackageDetailsSuccess = PackageDetails & {
+  docsLink: string;
+};
+
+export type PackageDocsResult = {
+  packageInfo: ParsedPackageInfo;
+} & (
+  | { error: string; details?: never }
+  | { details: PackageDetailsSuccess; error?: never }
+);
diff --git a/core/indexing/docs/suggestions/index.ts b/core/indexing/docs/suggestions/index.ts
index d3d68aca29..5699cd761c 100644
--- a/core/indexing/docs/suggestions/index.ts
+++ b/core/indexing/docs/suggestions/index.ts
@@ -1,12 +1,12 @@
-// write me an interface PackageCrawler that contains:
-// 1. property `language` to store a given language like "python" or "typescript"
-// 2. has a method `getPackageFiles` which takes a list of file names and decides which ones match package/dependency files (e.g. package.json for typescript, requirements.txt for python, etc)
-// 3. has a method `parsePackageFile` which returns a list of package name and version from a relevant package file, in a standardized format like semver
-// 4. has a method `getDocumentationLink` to check for documentation link for a given package (e.g. GET `https://registry.npmjs.org/<package>` and find docs field for typescript, documentation link in the package metadata for PyPi, etc.)
-// Then, write typescript classes to implement this typescript interface for the languages "python" and "typescript"
-
-import { IDE } from "../../..";
+import {
+  FilePathAndName,
+  IDE,
+  PackageDetails,
+  PackageDocsResult,
+  ParsedPackageInfo,
+} from "../../..";
 import { walkDir } from "../../walkDir";
+
 import { PythonPackageCrawler } from "./packageCrawlers/Python";
 import { TypeScriptPackageCrawler } from "./packageCrawlers/TsJs";
 
@@ -14,19 +14,14 @@ const PACKAGE_CRAWLERS = [TypeScriptPackageCrawler, PythonPackageCrawler];
 
 export interface PackageCrawler {
   language: string;
-  getPackageFiles(fileNames: string[]): string[];
-  parsePackageFile(fileContent: string, filePath: string): PackageInfo[];
-  getDocumentationLink(packageName: string): Promise<PackageDocsResult>;
+  getPackageFiles(files: FilePathAndName[]): FilePathAndName[];
+  parsePackageFile(
+    file: FilePathAndName,
+    contents: string,
+  ): ParsedPackageInfo[];
+  getPackageDetails(packageInfo: ParsedPackageInfo): Promise<PackageDetails>;
 }
 
-export type PackageInfo = {
-  name: string;
-  version: string;
-  foundInFilepath: string;
-};
-export type PackageDocsResult = PackageInfo &
-  ({ error: string; link?: never } | { link: string; error?: never });
-
 export async function getAllSuggestedDocs(ide: IDE) {
   const workspaceDirs = await ide.getWorkspaceDirs();
   const results = await Promise.all(
@@ -35,16 +30,26 @@ export async function getAllSuggestedDocs(ide: IDE) {
     }),
   );
   const allPaths = results.flat(); // TODO only get files, not dirs. Not critical for now
-  const allFiles = allPaths.map((path) => path.split("/").pop()!);
-  const packageFilesByLanguage: Record<string, string[]> = {};
+  const allFiles = allPaths.map((path) => ({
+    path,
+    name: path.split(/[\\/]/).pop()!,
+  }));
+
+  // Build map of language -> package files
+  const packageFilesByLanguage: Record<string, FilePathAndName[]> = {};
   for (const Crawler of PACKAGE_CRAWLERS) {
     const crawler = new Crawler();
     const packageFilePaths = crawler.getPackageFiles(allFiles);
     packageFilesByLanguage[crawler.language] = packageFilePaths;
   }
 
+  // Get file contents for all unique package files
   const uniqueFilePaths = Array.from(
-    new Set(Object.values(packageFilesByLanguage).flat()),
+    new Set(
+      Object.values(packageFilesByLanguage).flatMap((files) =>
+        files.map((file) => file.path),
+      ),
+    ),
   );
   const fileContentsArray = await Promise.all(
     uniqueFilePaths.map(async (path) => {
@@ -56,40 +61,68 @@ export async function getAllSuggestedDocs(ide: IDE) {
     fileContentsArray.map(({ path, contents }) => [path, contents]),
   );
 
-  const packagesByLanguage: Record<string, PackageInfo[]> = {};
+  // Parse package files and build map of language -> packages
+  const packagesByLanguage: Record<string, ParsedPackageInfo[]> = {};
   PACKAGE_CRAWLERS.forEach((Crawler) => {
     const crawler = new Crawler();
     const packageFiles = packageFilesByLanguage[crawler.language];
     packageFiles.forEach((file) => {
-      const contents = fileContents.get(file);
+      const contents = fileContents.get(file.path);
       if (!contents) {
         return;
       }
-      const packages = crawler.parsePackageFile(contents, file);
+      const packages = crawler.parsePackageFile(file, contents);
       if (!packagesByLanguage[crawler.language]) {
         packagesByLanguage[crawler.language] = [];
       }
       packagesByLanguage[crawler.language].push(...packages);
     });
   });
-  console.log(packagesByLanguage);
-  //   const allPackages = await Promise.all(
-  //     PACKAGE_CRAWLERS.map(async (Crawler) => {
-  //         const crawler = new Crawler();
-  //         const packageInfos = uniqueFileContents
-  //             .filter(({ path }) => languageToFilePaths[crawler.language].includes(path))
-  //             .map(({ path, contents }) => crawler.parsePackageFile(contents, path))
-  //             .flat();
-  //         return packageInfos;
-  //     })
-  //   );
-  //   for (const Crawler of PACKAGE_CRAWLERS) {
-  //     const crawler = new Crawler();
-  //     const packages = crawler.parsePackageFile()
-  //     languageToFilePaths[crawler.language] = packageFilePaths;
-  //   }
+
+  // Get documentation links for all packages
+  const docsByLanguage: Record<string, PackageDocsResult[]> = {};
+  await Promise.all(
+    PACKAGE_CRAWLERS.map(async (Crawler) => {
+      const crawler = new Crawler();
+      const packages = packagesByLanguage[crawler.language];
+      docsByLanguage[crawler.language] = await Promise.all(
+        packages.map(async (packageInfo) => {
+          try {
+            const details = await crawler.getPackageDetails(packageInfo);
+            if (!details.docsLink) {
+              return {
+                packageInfo,
+                error: `No documentation link found for ${packageInfo.name}`,
+              };
+            }
+            return {
+              packageInfo,
+              details: {
+                ...details,
+                docsLink: details.docsLink,
+              },
+            };
+          } catch (error) {
+            return {
+              packageInfo,
+              error: `Error getting package details for ${name}`,
+            };
+          }
+        }),
+      );
+    }),
+  );
+
+  return docsByLanguage;
 }
 
+// write me an interface PackageCrawler that contains:
+// 1. property `language` to store a given language like "python" or "typescript"
+// 2. has a method `getPackageFiles` which takes a list of file names and decides which ones match package/dependency files (e.g. package.json for typescript, requirements.txt for python, etc)
+// 3. has a method `parsePackageFile` which returns a list of package name and version from a relevant package file, in a standardized format like semver
+// 4. has a method `getDocumentationLink` to check for documentation link for a given package (e.g. GET `https://registry.npmjs.org/<package>` and find docs field for typescript, documentation link in the package metadata for PyPi, etc.)
+// Then, write typescript classes to implement this typescript interface for the languages "python" and "typescript"
+
 // I want to present the user with a list of dependencies and allow them to select which ones to index (embed) documentation for.
 // In order to prevent duplicate file reads, the process will be like this:
 // 1. take in a list of filepaths called `filepaths`
diff --git a/core/indexing/docs/suggestions/packageCrawlers/Python.ts b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
index ac06997663..233e444285 100644
--- a/core/indexing/docs/suggestions/packageCrawlers/Python.ts
+++ b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
@@ -1,33 +1,50 @@
-import { PackageCrawler, PackageDocsResult, PackageInfo } from "..";
+import { PackageCrawler } from "..";
+import {
+  FilePathAndName,
+  PackageDetails,
+  ParsedPackageInfo,
+} from "../../../..";
 
 export class PythonPackageCrawler implements PackageCrawler {
   language = "python";
 
-  getPackageFiles(fileNames: string[]): string[] {
+  getPackageFiles(files: FilePathAndName[]): FilePathAndName[] {
     // For Python, we typically look for files like requirements.txt or Pipfile
-    return fileNames.filter(
-      (fileName) => fileName === "requirements.txt" || fileName === "Pipfile",
+    return files.filter(
+      (file) => file.name === "requirements.txt" || file.name === "Pipfile",
     );
   }
 
-  parsePackageFile(fileContent: string, filepath: string): PackageInfo[] {
+  parsePackageFile(
+    file: FilePathAndName,
+    contents: string,
+  ): ParsedPackageInfo[] {
     // Assume the fileContent is a string from a requirements.txt formatted file
-    return fileContent
+    return contents
       .split("\n")
       .map((line) => {
         const [name, version] = line.split("==");
-        return { name, version, foundInFilepath: filepath };
+        return { name, version, packageFile: file, language: this.language };
       })
       .filter((pkg) => pkg.name && pkg.version);
   }
 
-  async getDocumentationLink(packageName: string): Promise<PackageDocsResult> {
+  async getPackageDetails(
+    packageInfo: ParsedPackageInfo,
+  ): Promise<PackageDetails> {
     // Fetch metadata from PyPI to find the documentation link
-    const response = await fetch(`https://pypi.org/pypi/${packageName}/json`);
+
+    const response = await fetch(
+      `https://pypi.org/pypi/${packageInfo.name}/json`,
+    );
     if (!response.ok) {
-      throw new Error(`Could not fetch data for package ${packageName}`);
+      throw new Error(`Could not fetch data for package ${packageInfo.name}`);
     }
     const data = await response.json();
-    return data.info.project_urls?.Documentation;
+    return {
+      docsLink: data.info.project_urls?.Documentation as string | undefined,
+      // title: data.info.name,
+      // description: data.info.summary as string | undefined,
+    };
   }
 }
diff --git a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
index fb073e2a1d..107d9ac2a1 100644
--- a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
+++ b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
@@ -1,33 +1,49 @@
-import { PackageCrawler, PackageDocsResult, PackageInfo } from "..";
+import { PackageCrawler } from "..";
+import {
+  FilePathAndName,
+  PackageDetails,
+  ParsedPackageInfo,
+} from "../../../..";
 
 export class TypeScriptPackageCrawler implements PackageCrawler {
-  language = "typescript";
+  language = "js-ts";
 
-  getPackageFiles(fileNames: string[]): string[] {
-    // For TypeScript, we look for package.json file
-    return fileNames.filter((fileName) => fileName === "package.json");
+  getPackageFiles(files: FilePathAndName[]): FilePathAndName[] {
+    // For Javascript/TypeScript, we look for package.json file
+    return files.filter((file) => file.name === "package.json");
   }
 
-  parsePackageFile(fileContent: string, filepath: string): PackageInfo[] {
+  parsePackageFile(
+    file: FilePathAndName,
+    contents: string,
+  ): ParsedPackageInfo[] {
     // Parse the package.json content
-    const jsonData = JSON.parse(fileContent) as Record<string, Object>;
+    const jsonData = JSON.parse(contents) as Record<string, Object>;
     const dependencies = Object.entries(jsonData.dependencies || {}).concat(
       Object.entries(jsonData.devDependencies || {}),
     );
     return dependencies.map(([name, version]) => ({
       name,
       version,
-      foundInFilepath: filepath,
+      packageFile: file,
+      language: this.language,
     }));
   }
 
-  async getDocumentationLink(packageName: string): Promise<PackageDocsResult> {
+  async getPackageDetails(
+    packageInfo: ParsedPackageInfo,
+  ): Promise<PackageDetails> {
+    const { name } = packageInfo;
     // Fetch metadata from the NPM registry to find the documentation link
-    const response = await fetch(`https://registry.npmjs.org/${packageName}`);
+    const response = await fetch(`https://registry.npmjs.org/${name}`);
     if (!response.ok) {
-      throw new Error(`Could not fetch data for package ${packageName}`);
+      throw new Error(`Could not fetch data for package ${name}`);
     }
     const data = await response.json();
-    return data.homepage;
+    return {
+      docsLink: data.homepage as string | undefined,
+      // title: data.name,
+      // description: data.description as string | undefined,
+    };
   }
 }

From abd46dfb4ff1aea84e94922fdda5d6fce23c6edf Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Thu, 21 Nov 2024 12:52:25 -0800
Subject: [PATCH 03/17] docs suggestions: packages in GUI

---
 core/core.ts                                 |  3 +-
 core/index.d.ts                              |  3 ++
 core/indexing/docs/suggestions/index.ts      |  4 +--
 core/protocol/passThrough.ts                 |  1 +
 core/protocol/webview.ts                     |  2 ++
 gui/src/components/dialogs/AddDocsDialog.tsx | 37 ++++++++++++++++++--
 gui/src/hooks/useSetup.ts                    |  6 ++++
 gui/src/redux/slices/stateSlice.ts           | 11 ++++++
 8 files changed, 61 insertions(+), 6 deletions(-)

diff --git a/core/core.ts b/core/core.ts
index d080ec34b0..d81df8a379 100644
--- a/core/core.ts
+++ b/core/core.ts
@@ -739,7 +739,8 @@ export class Core {
       return this.docsService.initStatuses();
     });
     on("docs/getSuggestedDocs", async (msg) => {
-      getAllSuggestedDocs(this.ide);
+      const suggestedDocs = await getAllSuggestedDocs(this.ide);
+      this.messenger.send("docs/suggestions", suggestedDocs);
     });
     //
 
diff --git a/core/index.d.ts b/core/index.d.ts
index de1adb8934..d6b2c1970c 100644
--- a/core/index.d.ts
+++ b/core/index.d.ts
@@ -1219,3 +1219,6 @@ export type PackageDocsResult = {
   | { error: string; details?: never }
   | { details: PackageDetailsSuccess; error?: never }
 );
+
+// language -> package -> package info
+export type DocsSuggestions = Record<string, PackageDocsResult[]>;
diff --git a/core/indexing/docs/suggestions/index.ts b/core/indexing/docs/suggestions/index.ts
index 5699cd761c..3ee4fb6426 100644
--- a/core/indexing/docs/suggestions/index.ts
+++ b/core/indexing/docs/suggestions/index.ts
@@ -1,4 +1,5 @@
 import {
+  DocsSuggestions,
   FilePathAndName,
   IDE,
   PackageDetails,
@@ -80,7 +81,7 @@ export async function getAllSuggestedDocs(ide: IDE) {
   });
 
   // Get documentation links for all packages
-  const docsByLanguage: Record<string, PackageDocsResult[]> = {};
+  const docsByLanguage: DocsSuggestions = {};
   await Promise.all(
     PACKAGE_CRAWLERS.map(async (Crawler) => {
       const crawler = new Crawler();
@@ -112,7 +113,6 @@ export async function getAllSuggestedDocs(ide: IDE) {
       );
     }),
   );
-
   return docsByLanguage;
 }
 
diff --git a/core/protocol/passThrough.ts b/core/protocol/passThrough.ts
index ebe2a86977..2bd8de2c18 100644
--- a/core/protocol/passThrough.ts
+++ b/core/protocol/passThrough.ts
@@ -75,4 +75,5 @@ export const CORE_TO_WEBVIEW_PASS_THROUGH: (keyof ToWebviewFromCoreProtocol)[] =
     "getWebviewHistoryLength",
     "signInToControlPlane",
     "openDialogMessage",
+    "docs/suggestions",
   ];
diff --git a/core/protocol/webview.ts b/core/protocol/webview.ts
index 0dcdb047f6..bf90497c8a 100644
--- a/core/protocol/webview.ts
+++ b/core/protocol/webview.ts
@@ -2,6 +2,7 @@ import { ConfigValidationError } from "../config/validation.js";
 
 import type {
   ContextItemWithId,
+  DocsSuggestions,
   IndexingProgressUpdate,
   IndexingStatus,
 } from "../index.js";
@@ -25,4 +26,5 @@ export type ToWebviewFromIdeOrCoreProtocol = {
   getWebviewHistoryLength: [undefined, number];
   signInToControlPlane: [undefined, void];
   openDialogMessage: ["account", void];
+  "docs/suggestions": [DocsSuggestions, void];
 };
diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx
index 0981948869..c21aca5a6c 100644
--- a/gui/src/components/dialogs/AddDocsDialog.tsx
+++ b/gui/src/components/dialogs/AddDocsDialog.tsx
@@ -5,7 +5,7 @@ import {
 } from "@heroicons/react/24/outline";
 import { SiteIndexingConfig } from "core";
 import { usePostHog } from "posthog-js/react";
-import { useContext, useLayoutEffect, useRef, useState } from "react";
+import { useContext, useLayoutEffect, useMemo, useRef, useState } from "react";
 import { useDispatch, useSelector } from "react-redux";
 import { Button, HelperText, Input, lightGray, SecondaryButton } from "..";
 import { IdeMessengerContext } from "../../context/IdeMessenger";
@@ -15,6 +15,7 @@ import {
 } from "../../redux/slices/uiStateSlice";
 import { RootState } from "../../redux/store";
 import IndexingStatusViewer from "../indexing/IndexingStatus";
+import { C } from "core/autocomplete/constants/AutocompleteLanguageInfo";
 
 function AddDocsDialog() {
   const posthog = usePostHog();
@@ -34,6 +35,21 @@ function AddDocsDialog() {
     (store: RootState) => store.state.indexing.statuses,
   );
 
+  const docsSuggestions = useSelector(
+    (store: RootState) => store.state.docsSuggestions,
+  );
+
+  const docsByLanguage = useMemo(() => {
+    console.log(docsSuggestions);
+    const languages = Object.keys(docsSuggestions);
+    return languages.map((language) => {
+      return {
+        language,
+        packages: docsSuggestions[language],
+      };
+    });
+  }, [docsSuggestions]);
+
   const isFormValid = startUrl && title;
 
   useLayoutEffect(() => {
@@ -118,8 +134,23 @@ function AddDocsDialog() {
   return (
     <div className="p-4">
       <div className="mb-8">
-        <h1>Add a documentation site</h1>
-
+        <h1>Add documentation</h1>
+        {docsByLanguage.map(({ language, packages }) => {
+          return (
+            <div key={language}>
+              <h1>{language}</h1>
+              <div>
+                {packages.map((pkg) => {
+                  return (
+                    <div>
+                      <p>{pkg.packageInfo.name}</p>
+                    </div>
+                  );
+                })}
+              </div>
+            </div>
+          );
+        })}
         <p>
           Continue pre-indexes many common documentation sites, but if there's
           one you don't see in the dropdown, enter the URL here.
diff --git a/gui/src/hooks/useSetup.ts b/gui/src/hooks/useSetup.ts
index f183798890..ecc7a7ac92 100644
--- a/gui/src/hooks/useSetup.ts
+++ b/gui/src/hooks/useSetup.ts
@@ -11,6 +11,7 @@ import {
   setInactive,
   setSelectedProfileId,
   setTTSActive,
+  updateDocsSuggestions,
   updateIndexingStatus,
 } from "../redux/slices/stateSlice";
 import { RootState } from "../redux/store";
@@ -85,6 +86,7 @@ function useSetup(dispatch: Dispatch) {
   // ON LOAD
   useEffect(() => {
     ideMessenger.post("docs/getSuggestedDocs", undefined);
+
     // Override persisted state
     dispatch(setInactive());
 
@@ -116,6 +118,10 @@ function useSetup(dispatch: Dispatch) {
     }
   }, []);
 
+  useWebviewListener("docs/suggestions", async (data) => {
+    dispatch(updateDocsSuggestions(data));
+  });
+
   const { streamResponse } = useChatHandler(dispatch, ideMessenger);
 
   const defaultModelTitle = useSelector(
diff --git a/gui/src/redux/slices/stateSlice.ts b/gui/src/redux/slices/stateSlice.ts
index 770e9ce698..e965455c4f 100644
--- a/gui/src/redux/slices/stateSlice.ts
+++ b/gui/src/redux/slices/stateSlice.ts
@@ -5,6 +5,7 @@ import {
   ChatMessage,
   Checkpoint,
   ContextItemWithId,
+  DocsSuggestions,
   FileSymbolMap,
   IndexingStatus,
   PersistedSessionInfo,
@@ -47,6 +48,7 @@ type State = {
   };
   streamAborter: AbortController;
   isMultifileEdit: boolean;
+  docsSuggestions: DocsSuggestions;
 };
 
 const initialState: State = {
@@ -89,6 +91,7 @@ const initialState: State = {
     },
   },
   streamAborter: new AbortController(),
+  docsSuggestions: {},
 };
 
 export const stateSlice = createSlice({
@@ -447,6 +450,13 @@ export const stateSlice = createSlice({
         [payload.type]: payload.hidden,
       };
     },
+    updateDocsSuggestions: (
+      state,
+      { payload }: PayloadAction<DocsSuggestions>,
+    ) => {
+      console.log("FROM REDUX", payload);
+      state.docsSuggestions = payload;
+    },
   },
 });
 
@@ -480,6 +490,7 @@ export const {
   updateIndexingStatus,
   setIndexingChatPeekHidden,
   abortStream,
+  updateDocsSuggestions,
 } = stateSlice.actions;
 
 export default stateSlice.reducer;

From 53f3638b8f0ec703579f3136f7293d4d88ea8a28 Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Thu, 21 Nov 2024 16:37:28 -0800
Subject: [PATCH 04/17] suggested docs continued

---
 core/index.d.ts                               | 11 +++++--
 core/indexing/docs/suggestions/index.ts       | 30 ++++++++++++-----
 .../suggestions/packageCrawlers/Python.ts     | 29 +++++++++++++----
 .../docs/suggestions/packageCrawlers/TsJs.ts  | 24 +++++++++++---
 gui/src/components/dialogs/AddDocsDialog.tsx  | 32 ++++---------------
 gui/src/components/dialogs/SuggestedDocs.tsx  | 18 +++++++++++
 gui/src/redux/slices/stateSlice.ts            |  9 +++---
 7 files changed, 100 insertions(+), 53 deletions(-)
 create mode 100644 gui/src/components/dialogs/SuggestedDocs.tsx

diff --git a/core/index.d.ts b/core/index.d.ts
index d6b2c1970c..328462e70e 100644
--- a/core/index.d.ts
+++ b/core/index.d.ts
@@ -1196,6 +1196,12 @@ export interface FilePathAndName {
   path: string;
   name: string;
 }
+
+export interface PackageFilePathAndName extends FilePathAndName {
+  language: string; // e.g. javascript
+  registry: string; // e.g. npm
+}
+
 export type ParsedPackageInfo = {
   language: string;
   name: string;
@@ -1207,6 +1213,8 @@ export type PackageDetails = {
   docsLink?: string;
   title?: string;
   description?: string;
+  repo?: string;
+  license?: string;
 };
 
 export type PackageDetailsSuccess = PackageDetails & {
@@ -1219,6 +1227,3 @@ export type PackageDocsResult = {
   | { error: string; details?: never }
   | { details: PackageDetailsSuccess; error?: never }
 );
-
-// language -> package -> package info
-export type DocsSuggestions = Record<string, PackageDocsResult[]>;
diff --git a/core/indexing/docs/suggestions/index.ts b/core/indexing/docs/suggestions/index.ts
index 3ee4fb6426..0f86f53902 100644
--- a/core/indexing/docs/suggestions/index.ts
+++ b/core/indexing/docs/suggestions/index.ts
@@ -1,9 +1,9 @@
 import {
-  DocsSuggestions,
+  PackageDocsResult,
   FilePathAndName,
+  PackageFilePathAndName,
   IDE,
   PackageDetails,
-  PackageDocsResult,
   ParsedPackageInfo,
 } from "../../..";
 import { walkDir } from "../../walkDir";
@@ -15,9 +15,9 @@ const PACKAGE_CRAWLERS = [TypeScriptPackageCrawler, PythonPackageCrawler];
 
 export interface PackageCrawler {
   language: string;
-  getPackageFiles(files: FilePathAndName[]): FilePathAndName[];
+  getPackageFiles(files: FilePathAndName[]): PackageFilePathAndName[];
   parsePackageFile(
-    file: FilePathAndName,
+    file: PackageFilePathAndName,
     contents: string,
   ): ParsedPackageInfo[];
   getPackageDetails(packageInfo: ParsedPackageInfo): Promise<PackageDetails>;
@@ -37,7 +37,7 @@ export async function getAllSuggestedDocs(ide: IDE) {
   }));
 
   // Build map of language -> package files
-  const packageFilesByLanguage: Record<string, FilePathAndName[]> = {};
+  const packageFilesByLanguage: Record<string, PackageFilePathAndName[]> = {};
   for (const Crawler of PACKAGE_CRAWLERS) {
     const crawler = new Crawler();
     const packageFilePaths = crawler.getPackageFiles(allFiles);
@@ -80,13 +80,26 @@ export async function getAllSuggestedDocs(ide: IDE) {
     });
   });
 
+  // Deduplicate packages per language
+  // TODO - this is where you would allow docs for different versions
+  // by e.g. using "name-version" as the map key instead of just name
+  // For now have not allowed
+  const languages = Object.keys(packagesByLanguage);
+  languages.forEach((language) => {
+    const packages = packagesByLanguage[language];
+    const uniquePackages = Array.from(
+      new Map(packages.map((pkg) => [pkg.name, pkg])).values(),
+    );
+    packagesByLanguage[language] = uniquePackages;
+  });
+
   // Get documentation links for all packages
-  const docsByLanguage: DocsSuggestions = {};
+  const allDocsResults: PackageDocsResult[] = [];
   await Promise.all(
     PACKAGE_CRAWLERS.map(async (Crawler) => {
       const crawler = new Crawler();
       const packages = packagesByLanguage[crawler.language];
-      docsByLanguage[crawler.language] = await Promise.all(
+      const docsByLanguage = await Promise.all(
         packages.map(async (packageInfo) => {
           try {
             const details = await crawler.getPackageDetails(packageInfo);
@@ -111,9 +124,10 @@ export async function getAllSuggestedDocs(ide: IDE) {
           }
         }),
       );
+      allDocsResults.push(...docsByLanguage);
     }),
   );
-  return docsByLanguage;
+  return allDocsResults;
 }
 
 // write me an interface PackageCrawler that contains:
diff --git a/core/indexing/docs/suggestions/packageCrawlers/Python.ts b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
index 233e444285..28b893b66b 100644
--- a/core/indexing/docs/suggestions/packageCrawlers/Python.ts
+++ b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
@@ -2,17 +2,24 @@ import { PackageCrawler } from "..";
 import {
   FilePathAndName,
   PackageDetails,
+  PackageFilePathAndName,
   ParsedPackageInfo,
 } from "../../../..";
 
 export class PythonPackageCrawler implements PackageCrawler {
   language = "python";
 
-  getPackageFiles(files: FilePathAndName[]): FilePathAndName[] {
+  getPackageFiles(files: FilePathAndName[]): PackageFilePathAndName[] {
     // For Python, we typically look for files like requirements.txt or Pipfile
-    return files.filter(
-      (file) => file.name === "requirements.txt" || file.name === "Pipfile",
-    );
+    return files
+      .filter(
+        (file) => file.name === "requirements.txt" || file.name === "Pipfile",
+      )
+      .map((file) => ({
+        ...file,
+        language: this.language,
+        registry: "pypi",
+      }));
   }
 
   parsePackageFile(
@@ -41,10 +48,18 @@ export class PythonPackageCrawler implements PackageCrawler {
       throw new Error(`Could not fetch data for package ${packageInfo.name}`);
     }
     const data = await response.json();
+    const homePage = data?.info?.home_page as string | undefined;
+
     return {
-      docsLink: data.info.project_urls?.Documentation as string | undefined,
-      // title: data.info.name,
-      // description: data.info.summary as string | undefined,
+      docsLink:
+        (data?.info?.project_urls?.Documentation as string | undefined) ??
+        homePage,
+      title: data?.info?.name as string | undefined,
+      description: data?.info?.summary as string | undefined,
+      repo:
+        (data?.info?.project_urls?.Repository as string | undefined) ??
+        homePage,
+      license: data?.info?.license as string | undefined,
     };
   }
 }
diff --git a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
index 107d9ac2a1..c16b8872e8 100644
--- a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
+++ b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
@@ -2,15 +2,22 @@ import { PackageCrawler } from "..";
 import {
   FilePathAndName,
   PackageDetails,
+  PackageFilePathAndName,
   ParsedPackageInfo,
 } from "../../../..";
 
 export class TypeScriptPackageCrawler implements PackageCrawler {
   language = "js-ts";
 
-  getPackageFiles(files: FilePathAndName[]): FilePathAndName[] {
+  getPackageFiles(files: FilePathAndName[]): PackageFilePathAndName[] {
     // For Javascript/TypeScript, we look for package.json file
-    return files.filter((file) => file.name === "package.json");
+    return files
+      .filter((file) => file.name === "package.json")
+      .map((file) => ({
+        ...file,
+        language: this.language,
+        registry: "npm",
+      }));
   }
 
   parsePackageFile(
@@ -22,7 +29,10 @@ export class TypeScriptPackageCrawler implements PackageCrawler {
     const dependencies = Object.entries(jsonData.dependencies || {}).concat(
       Object.entries(jsonData.devDependencies || {}),
     );
-    return dependencies.map(([name, version]) => ({
+    const filtered = dependencies.filter(
+      ([name, version]) => !name.startsWith("@types/"),
+    );
+    return filtered.map(([name, version]) => ({
       name,
       version,
       packageFile: file,
@@ -42,8 +52,12 @@ export class TypeScriptPackageCrawler implements PackageCrawler {
     const data = await response.json();
     return {
       docsLink: data.homepage as string | undefined,
-      // title: data.name,
-      // description: data.description as string | undefined,
+      title: name, // package.json doesn't have specific title field
+      description: data.description as string | undefined,
+      repo: Array.isArray(data.repository)
+        ? (data.respository[0]?.url as string | undefined)
+        : undefined,
+      license: data.license as string | undefined,
     };
   }
 }
diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx
index c21aca5a6c..7dcc478fc6 100644
--- a/gui/src/components/dialogs/AddDocsDialog.tsx
+++ b/gui/src/components/dialogs/AddDocsDialog.tsx
@@ -15,7 +15,6 @@ import {
 } from "../../redux/slices/uiStateSlice";
 import { RootState } from "../../redux/store";
 import IndexingStatusViewer from "../indexing/IndexingStatus";
-import { C } from "core/autocomplete/constants/AutocompleteLanguageInfo";
 
 function AddDocsDialog() {
   const posthog = usePostHog();
@@ -39,17 +38,6 @@ function AddDocsDialog() {
     (store: RootState) => store.state.docsSuggestions,
   );
 
-  const docsByLanguage = useMemo(() => {
-    console.log(docsSuggestions);
-    const languages = Object.keys(docsSuggestions);
-    return languages.map((language) => {
-      return {
-        language,
-        packages: docsSuggestions[language],
-      };
-    });
-  }, [docsSuggestions]);
-
   const isFormValid = startUrl && title;
 
   useLayoutEffect(() => {
@@ -135,22 +123,16 @@ function AddDocsDialog() {
     <div className="p-4">
       <div className="mb-8">
         <h1>Add documentation</h1>
-        {docsByLanguage.map(({ language, packages }) => {
+        {/* {docsSuggestions.map((docsResult) => {
+          const { error, details } = docsResult;
+          const { language, name, version } = docsResult.packageInfo;
           return (
-            <div key={language}>
-              <h1>{language}</h1>
-              <div>
-                {packages.map((pkg) => {
-                  return (
-                    <div>
-                      <p>{pkg.packageInfo.name}</p>
-                    </div>
-                  );
-                })}
-              </div>
+            <div key={`${language}-${name}-${version}`}>
+              <p className="m-0 p-0">{`${name}`}</p>
+              <p className="m-0 p-0">{error ? "error" : details.docsLink}</p>
             </div>
           );
-        })}
+        })} */}
         <p>
           Continue pre-indexes many common documentation sites, but if there's
           one you don't see in the dropdown, enter the URL here.
diff --git a/gui/src/components/dialogs/SuggestedDocs.tsx b/gui/src/components/dialogs/SuggestedDocs.tsx
new file mode 100644
index 0000000000..1540ea92f9
--- /dev/null
+++ b/gui/src/components/dialogs/SuggestedDocs.tsx
@@ -0,0 +1,18 @@
+import { PackageDocsResult } from "core";
+
+interface SuggestedDocProps {
+  docResult: PackageDocsResult;
+}
+const SuggestedDoc = ({ docResult }: SuggestedDocProps) => {
+  return <div>SuggestedDoc</div>;
+};
+
+interface SuggestedDocsListProps {
+  docs: PackageDocsResult[];
+  //   on
+}
+const SuggestedDocsList = ({ docs }: SuggestedDocsListProps) => {
+  return <div></div>;
+};
+
+export default SuggestedDocsList;
diff --git a/gui/src/redux/slices/stateSlice.ts b/gui/src/redux/slices/stateSlice.ts
index e965455c4f..2feaf80651 100644
--- a/gui/src/redux/slices/stateSlice.ts
+++ b/gui/src/redux/slices/stateSlice.ts
@@ -5,7 +5,7 @@ import {
   ChatMessage,
   Checkpoint,
   ContextItemWithId,
-  DocsSuggestions,
+  PackageDocsResult,
   FileSymbolMap,
   IndexingStatus,
   PersistedSessionInfo,
@@ -48,7 +48,7 @@ type State = {
   };
   streamAborter: AbortController;
   isMultifileEdit: boolean;
-  docsSuggestions: DocsSuggestions;
+  docsSuggestions: PackageDocsResult[];
 };
 
 const initialState: State = {
@@ -91,7 +91,7 @@ const initialState: State = {
     },
   },
   streamAborter: new AbortController(),
-  docsSuggestions: {},
+  docsSuggestions: [],
 };
 
 export const stateSlice = createSlice({
@@ -452,9 +452,8 @@ export const stateSlice = createSlice({
     },
     updateDocsSuggestions: (
       state,
-      { payload }: PayloadAction<DocsSuggestions>,
+      { payload }: PayloadAction<PackageDocsResult[]>,
     ) => {
-      console.log("FROM REDUX", payload);
       state.docsSuggestions = payload;
     },
   },

From 48806f3f0831de8cf67546ef77467548187641cc Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Thu, 21 Nov 2024 18:07:37 -0800
Subject: [PATCH 05/17] docs suggestions p4

---
 core/core.ts                                  |   8 +-
 core/index.d.ts                               |   7 +-
 core/indexing/docs/suggestions/index.ts       |  36 ++--
 .../suggestions/packageCrawlers/Python.ts     |   9 +-
 .../docs/suggestions/packageCrawlers/TsJs.ts  |  27 ++-
 gui/src/components/dialogs/AddDocsDialog.tsx  | 186 ++++++++++++------
 gui/src/components/index.ts                   |  10 +
 .../components/indexing/IndexingStatuses.tsx  |  27 ++-
 gui/tailwind.config.cjs                       |  11 +-
 9 files changed, 199 insertions(+), 122 deletions(-)

diff --git a/core/core.ts b/core/core.ts
index d81df8a379..b2be3478c3 100644
--- a/core/core.ts
+++ b/core/core.ts
@@ -19,6 +19,7 @@ import { ControlPlaneClient } from "./control-plane/client";
 import { streamDiffLines } from "./edit/streamDiffLines";
 import { CodebaseIndexer, PauseToken } from "./indexing/CodebaseIndexer";
 import DocsService from "./indexing/docs/DocsService";
+import { getAllSuggestedDocs } from "./indexing/docs/suggestions";
 import { defaultIgnoreFile } from "./indexing/ignore.js";
 import Ollama from "./llm/llms/Ollama";
 import { createNewPromptFileV2 } from "./promptFiles/v2/createNewPromptFile";
@@ -28,11 +29,7 @@ import { DevDataSqliteDb } from "./util/devdataSqlite";
 import { fetchwithRequestOptions } from "./util/fetchWithOptions";
 import { GlobalContext } from "./util/GlobalContext";
 import historyManager from "./util/history";
-import {
-  editConfigJson,
-  getConfigJsonPath,
-  setupInitialDotContinueDirectory,
-} from "./util/paths";
+import { editConfigJson, setupInitialDotContinueDirectory } from "./util/paths";
 import { Telemetry } from "./util/posthog";
 import { getSymbolsForManyFiles } from "./util/treeSitter";
 import { TTS } from "./util/tts";
@@ -40,7 +37,6 @@ import { TTS } from "./util/tts";
 import type { ContextItemId, IDE, IndexingProgressUpdate } from ".";
 import type { FromCoreProtocol, ToCoreProtocol } from "./protocol";
 import type { IMessenger, Message } from "./util/messenger";
-import { getAllSuggestedDocs } from "./indexing/docs/suggestions";
 
 export class Core {
   // implements IMessenger<ToCoreProtocol, FromCoreProtocol>
diff --git a/core/index.d.ts b/core/index.d.ts
index 328462e70e..5541584ebd 100644
--- a/core/index.d.ts
+++ b/core/index.d.ts
@@ -1198,15 +1198,14 @@ export interface FilePathAndName {
 }
 
 export interface PackageFilePathAndName extends FilePathAndName {
-  language: string; // e.g. javascript
-  registry: string; // e.g. npm
+  packageRegistry: string; // e.g. npm, pypi
 }
 
 export type ParsedPackageInfo = {
-  language: string;
   name: string;
+  packageFile: PackageFilePathAndName;
+  language: string;
   version: string;
-  packageFile: FilePathAndName;
 };
 
 export type PackageDetails = {
diff --git a/core/indexing/docs/suggestions/index.ts b/core/indexing/docs/suggestions/index.ts
index 0f86f53902..4e0c2104ac 100644
--- a/core/indexing/docs/suggestions/index.ts
+++ b/core/indexing/docs/suggestions/index.ts
@@ -9,12 +9,12 @@ import {
 import { walkDir } from "../../walkDir";
 
 import { PythonPackageCrawler } from "./packageCrawlers/Python";
-import { TypeScriptPackageCrawler } from "./packageCrawlers/TsJs";
+import { NodePackageCrawler } from "./packageCrawlers/TsJs";
 
-const PACKAGE_CRAWLERS = [TypeScriptPackageCrawler, PythonPackageCrawler];
+const PACKAGE_CRAWLERS = [NodePackageCrawler, PythonPackageCrawler];
 
 export interface PackageCrawler {
-  language: string;
+  packageRegistry: string;
   getPackageFiles(files: FilePathAndName[]): PackageFilePathAndName[];
   parsePackageFile(
     file: PackageFilePathAndName,
@@ -37,17 +37,17 @@ export async function getAllSuggestedDocs(ide: IDE) {
   }));
 
   // Build map of language -> package files
-  const packageFilesByLanguage: Record<string, PackageFilePathAndName[]> = {};
+  const packageFilesByRegistry: Record<string, PackageFilePathAndName[]> = {};
   for (const Crawler of PACKAGE_CRAWLERS) {
     const crawler = new Crawler();
     const packageFilePaths = crawler.getPackageFiles(allFiles);
-    packageFilesByLanguage[crawler.language] = packageFilePaths;
+    packageFilesByRegistry[crawler.packageRegistry] = packageFilePaths;
   }
 
   // Get file contents for all unique package files
   const uniqueFilePaths = Array.from(
     new Set(
-      Object.values(packageFilesByLanguage).flatMap((files) =>
+      Object.values(packageFilesByRegistry).flatMap((files) =>
         files.map((file) => file.path),
       ),
     ),
@@ -63,20 +63,20 @@ export async function getAllSuggestedDocs(ide: IDE) {
   );
 
   // Parse package files and build map of language -> packages
-  const packagesByLanguage: Record<string, ParsedPackageInfo[]> = {};
+  const packagesByRegistry: Record<string, ParsedPackageInfo[]> = {};
   PACKAGE_CRAWLERS.forEach((Crawler) => {
     const crawler = new Crawler();
-    const packageFiles = packageFilesByLanguage[crawler.language];
+    const packageFiles = packageFilesByRegistry[crawler.packageRegistry];
     packageFiles.forEach((file) => {
       const contents = fileContents.get(file.path);
       if (!contents) {
         return;
       }
       const packages = crawler.parsePackageFile(file, contents);
-      if (!packagesByLanguage[crawler.language]) {
-        packagesByLanguage[crawler.language] = [];
+      if (!packagesByRegistry[crawler.packageRegistry]) {
+        packagesByRegistry[crawler.packageRegistry] = [];
       }
-      packagesByLanguage[crawler.language].push(...packages);
+      packagesByRegistry[crawler.packageRegistry].push(...packages);
     });
   });
 
@@ -84,13 +84,13 @@ export async function getAllSuggestedDocs(ide: IDE) {
   // TODO - this is where you would allow docs for different versions
   // by e.g. using "name-version" as the map key instead of just name
   // For now have not allowed
-  const languages = Object.keys(packagesByLanguage);
+  const languages = Object.keys(packagesByRegistry);
   languages.forEach((language) => {
-    const packages = packagesByLanguage[language];
+    const packages = packagesByRegistry[language];
     const uniquePackages = Array.from(
       new Map(packages.map((pkg) => [pkg.name, pkg])).values(),
     );
-    packagesByLanguage[language] = uniquePackages;
+    packagesByRegistry[language] = uniquePackages;
   });
 
   // Get documentation links for all packages
@@ -98,8 +98,8 @@ export async function getAllSuggestedDocs(ide: IDE) {
   await Promise.all(
     PACKAGE_CRAWLERS.map(async (Crawler) => {
       const crawler = new Crawler();
-      const packages = packagesByLanguage[crawler.language];
-      const docsByLanguage = await Promise.all(
+      const packages = packagesByRegistry[crawler.packageRegistry];
+      const docsByRegistry = await Promise.all(
         packages.map(async (packageInfo) => {
           try {
             const details = await crawler.getPackageDetails(packageInfo);
@@ -119,12 +119,12 @@ export async function getAllSuggestedDocs(ide: IDE) {
           } catch (error) {
             return {
               packageInfo,
-              error: `Error getting package details for ${name}`,
+              error: `Error getting package details for ${packageInfo.name}`,
             };
           }
         }),
       );
-      allDocsResults.push(...docsByLanguage);
+      allDocsResults.push(...docsByRegistry);
     }),
   );
   return allDocsResults;
diff --git a/core/indexing/docs/suggestions/packageCrawlers/Python.ts b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
index 28b893b66b..d00045f976 100644
--- a/core/indexing/docs/suggestions/packageCrawlers/Python.ts
+++ b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
@@ -7,7 +7,7 @@ import {
 } from "../../../..";
 
 export class PythonPackageCrawler implements PackageCrawler {
-  language = "python";
+  packageRegistry = "pypi";
 
   getPackageFiles(files: FilePathAndName[]): PackageFilePathAndName[] {
     // For Python, we typically look for files like requirements.txt or Pipfile
@@ -17,13 +17,12 @@ export class PythonPackageCrawler implements PackageCrawler {
       )
       .map((file) => ({
         ...file,
-        language: this.language,
-        registry: "pypi",
+        packageRegistry: "pypi",
       }));
   }
 
   parsePackageFile(
-    file: FilePathAndName,
+    file: PackageFilePathAndName,
     contents: string,
   ): ParsedPackageInfo[] {
     // Assume the fileContent is a string from a requirements.txt formatted file
@@ -31,7 +30,7 @@ export class PythonPackageCrawler implements PackageCrawler {
       .split("\n")
       .map((line) => {
         const [name, version] = line.split("==");
-        return { name, version, packageFile: file, language: this.language };
+        return { name, version, packageFile: file, language: "python" };
       })
       .filter((pkg) => pkg.name && pkg.version);
   }
diff --git a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
index c16b8872e8..42c5007480 100644
--- a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
+++ b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
@@ -6,8 +6,8 @@ import {
   ParsedPackageInfo,
 } from "../../../..";
 
-export class TypeScriptPackageCrawler implements PackageCrawler {
-  language = "js-ts";
+export class NodePackageCrawler implements PackageCrawler {
+  packageRegistry = "npm";
 
   getPackageFiles(files: FilePathAndName[]): PackageFilePathAndName[] {
     // For Javascript/TypeScript, we look for package.json file
@@ -15,13 +15,12 @@ export class TypeScriptPackageCrawler implements PackageCrawler {
       .filter((file) => file.name === "package.json")
       .map((file) => ({
         ...file,
-        language: this.language,
-        registry: "npm",
+        packageRegistry: this.packageRegistry,
       }));
   }
 
   parsePackageFile(
-    file: FilePathAndName,
+    file: PackageFilePathAndName,
     contents: string,
   ): ParsedPackageInfo[] {
     // Parse the package.json content
@@ -29,14 +28,24 @@ export class TypeScriptPackageCrawler implements PackageCrawler {
     const dependencies = Object.entries(jsonData.dependencies || {}).concat(
       Object.entries(jsonData.devDependencies || {}),
     );
-    const filtered = dependencies.filter(
-      ([name, version]) => !name.startsWith("@types/"),
-    );
+
+    // Filter out types packages and check if typescript is present
+    let foundTypes = false;
+    const filtered = dependencies.filter(([name, _]) => {
+      if (name.startsWith("@types/")) {
+        foundTypes = true;
+        return false;
+      }
+      if (name.includes("typescript")) {
+        foundTypes = true;
+      }
+      return true;
+    });
     return filtered.map(([name, version]) => ({
       name,
       version,
       packageFile: file,
-      language: this.language,
+      language: foundTypes ? "ts" : "js",
     }));
   }
 
diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx
index 7dcc478fc6..ca6a8bceb1 100644
--- a/gui/src/components/dialogs/AddDocsDialog.tsx
+++ b/gui/src/components/dialogs/AddDocsDialog.tsx
@@ -2,8 +2,11 @@ import {
   CheckCircleIcon,
   ChevronDownIcon,
   ChevronUpIcon,
+  CodeBracketIcon,
+  InformationCircleIcon,
+  LinkIcon,
 } from "@heroicons/react/24/outline";
-import { SiteIndexingConfig } from "core";
+import { PackageDocsResult, SiteIndexingConfig } from "core";
 import { usePostHog } from "posthog-js/react";
 import { useContext, useLayoutEffect, useMemo, useRef, useState } from "react";
 import { useDispatch, useSelector } from "react-redux";
@@ -16,6 +19,8 @@ import {
 import { RootState } from "../../redux/store";
 import IndexingStatusViewer from "../indexing/IndexingStatus";
 
+import { ToolTip } from "../gui/Tooltip";
+
 function AddDocsDialog() {
   const posthog = usePostHog();
   const dispatch = useDispatch();
@@ -67,72 +72,125 @@ function AddDocsDialog() {
     posthog.capture("add_docs_gui", { url: startUrl });
   }
 
-  if (submittedConfig) {
-    const status = indexingStatuses[submittedConfig.startUrl];
-    return (
-      <div className="flex flex-col p-4">
-        <div className="flex flex-row items-center gap-2">
-          <CheckCircleIcon className="h-8 w-8" />
-          <h1>{`Docs added`}</h1>
-        </div>
-        <div className="flex flex-col gap-1 text-stone-500">
-          <p className="m-0 p-0">Title: {submittedConfig.title}</p>
-          <p className="m-0 p-0">Start URL: {submittedConfig.startUrl}</p>
-          {submittedConfig.rootUrl && (
-            <p className="m-0 p-0">Root URL: {submittedConfig.rootUrl}</p>
-          )}
-          {submittedConfig.maxDepth && (
-            <p className="m-0 p-0">Max depth: {submittedConfig.maxDepth}</p>
-          )}
-          {submittedConfig.faviconUrl && (
-            <p className="m-0 p-0">Favicon URL: {submittedConfig.faviconUrl}</p>
-          )}
-        </div>
-        {!!status && (
-          <div className="mt-4 flex flex-col divide-x-0 divide-y-2 divide-solid divide-zinc-700">
-            <p className="m-0 mb-5 p-0 leading-snug">{`Type "@docs" and select ${submittedConfig.title} to reference these docs once indexing is complete. Check indexing status from the "More" page.`}</p>
-            <div className="pt-1">
-              <IndexingStatusViewer status={status} />
-            </div>
-          </div>
-        )}
-        <div className="mt-4 flex flex-row items-center justify-end gap-4">
-          <SecondaryButton
-            className=""
-            onClick={() => {
-              setSubmittedConfig(undefined);
-            }}
-          >
-            Add another
-          </SecondaryButton>
-          <Button
-            className=""
-            onClick={() => {
-              dispatch(setDialogMessage(undefined));
-              dispatch(setShowDialog(false));
-            }}
-          >
-            Done
-          </Button>
-        </div>
-      </div>
-    );
-  }
+  const handleSelectSuggestion = (docsResult: PackageDocsResult) => {};
+
+  // if (submittedConfig) {
+  //   const status = indexingStatuses[submittedConfig.startUrl];
+  //   return (
+  //     <div className="flex flex-col p-4">
+  //       <div className="flex flex-row items-center gap-2">
+  //         <CheckCircleIcon className="h-8 w-8" />
+  //         <h1>{`Docs added`}</h1>
+  //       </div>
+  //       <div className="flex flex-col gap-1 text-stone-500">
+  //         <p className="m-0 p-0">Title: {submittedConfig.title}</p>
+  //         <p className="m-0 p-0">Start URL: {submittedConfig.startUrl}</p>
+  //         {submittedConfig.rootUrl && (
+  //           <p className="m-0 p-0">Root URL: {submittedConfig.rootUrl}</p>
+  //         )}
+  //         {submittedConfig.maxDepth && (
+  //           <p className="m-0 p-0">Max depth: {submittedConfig.maxDepth}</p>
+  //         )}
+  //         {submittedConfig.faviconUrl && (
+  //           <p className="m-0 p-0">Favicon URL: {submittedConfig.faviconUrl}</p>
+  //         )}
+  //       </div>
+  //       {!!status && (
+  //         <div className="mt-4 flex flex-col divide-x-0 divide-y-2 divide-solid divide-zinc-700">
+  //           <p className="m-0 mb-5 p-0 leading-snug">{`Type "@docs" and select ${submittedConfig.title} to reference these docs once indexing is complete. Check indexing status from the "More" page.`}</p>
+  //           <div className="pt-1">
+  //             <IndexingStatusViewer status={status} />
+  //           </div>
+  //         </div>
+  //       )}
+  //       <div className="mt-4 flex flex-row items-center justify-end gap-4">
+  //         <SecondaryButton
+  //           className=""
+  //           onClick={() => {
+  //             setSubmittedConfig(undefined);
+  //           }}
+  //         >
+  //           Add another
+  //         </SecondaryButton>
+  //         <Button
+  //           className=""
+  //           onClick={() => {
+  //             dispatch(setDialogMessage(undefined));
+  //             dispatch(setShowDialog(false));
+  //           }}
+  //         >
+  //           Done
+  //         </Button>
+  //       </div>
+  //     </div>
+  //   );
+  // }
 
   return (
     <div className="p-4">
       <div className="mb-8">
         <h1>Add documentation</h1>
-        {/* {docsSuggestions.map((docsResult) => {
-          const { error, details } = docsResult;
-          const { language, name, version } = docsResult.packageInfo;
-          return (
-            <div key={`${language}-${name}-${version}`}>
-              <p className="m-0 p-0">{`${name}`}</p>
-              <p className="m-0 p-0">{error ? "error" : details.docsLink}</p>
-            </div>
-          );
-        })} */}
+        {docsSuggestions.length ? (
+          <div className="no-scrollbar max-h-[300px] overflow-y-auto">
+            <table className="border-collapse p-0">
+              <thead className="bg-vsc-background sticky -top-1 font-bold">
+                <tr className="">
+                  <td className="pr-1">
+                    <CodeBracketIcon className="h-3.5 w-3.5" />
+                  </td>
+                  <td className="pr-1">Title</td>
+                  {/* <td className="pr-1">Version</td> */}
+                  <td className="pr-1">Start Link</td>
+                  <td></td>
+                </tr>
+              </thead>
+              <tbody className="p-0">
+                <tr className="whitespace-nowrap">Add docs</tr>
+                {docsSuggestions.map((docsResult) => {
+                  const { error, details } = docsResult;
+                  const { language, name, version } = docsResult.packageInfo;
+                  const id = `${language}-${name}-${version}`;
+                  return (
+                    <tr key={id} className="p-0">
+                      <td>
+                        <input type="checkbox"></input>
+                      </td>
+                      <td>{name}</td>
+                      {/* <td>{version}</td> */}
+                      <td className="">
+                        {error ? (
+                          <span className="text-vsc-input-border italic">
+                            No docs link found
+                          </span>
+                        ) : (
+                          <span className="flex flex-row items-center gap-2">
+                            <div>
+                              <LinkIcon className="h-2 w-2" />
+                            </div>
+                            <p className="lines lines-1 m-0 p-0">
+                              {details.docsLink}
+                            </p>
+                          </span>
+                        )}
+                      </td>
+                      <td>
+                        <InformationCircleIcon
+                          data-tooltip-id={id}
+                          className="text-vsc-foreground-muted h-3.5 w-3.5 cursor-help"
+                        />
+
+                        <ToolTip id={id} place="bottom">
+                          <p className="m-0 p-0">{`Version: ${version}`}</p>
+                          <p className="m-0 p-0">{`Found in ${docsResult.packageInfo.packageFile.path}`}</p>
+                        </ToolTip>
+                      </td>
+                    </tr>
+                  );
+                })}
+              </tbody>
+            </table>
+          </div>
+        ) : null}
         <p>
           Continue pre-indexes many common documentation sites, but if there's
           one you don't see in the dropdown, enter the URL here.
@@ -143,7 +201,7 @@ function AddDocsDialog() {
           so that you can ask questions.
         </p>
       </div>
-
+      {/* 
       <form onSubmit={onSubmit} className="flex flex-col space-y-4">
         <label>
           Title
@@ -219,7 +277,7 @@ function AddDocsDialog() {
             Submit
           </Button>
         </div>
-      </form>
+      </form> */}
     </div>
   );
 }
diff --git a/gui/src/components/index.ts b/gui/src/components/index.ts
index e380f3ef6f..74f27a44a3 100644
--- a/gui/src/components/index.ts
+++ b/gui/src/components/index.ts
@@ -4,6 +4,10 @@ import { getFontSize, isJetBrains } from "../util";
 export const VSC_INPUT_BACKGROUND_VAR = "--vscode-input-background";
 export const VSC_BACKGROUND_VAR = "--vscode-sideBar-background";
 export const VSC_FOREGROUND_VAR = "--vscode-editor-foreground";
+export const VSC_FOREGROUND_MUTED_VAR = "--vscode-foreground-muted";
+export const VSC_DESCRIPTION_FOREGROUND = "--vscode-descriptionForeground";
+export const VSC_INPUT_PLACEHOLDER_FOREGROUND =
+  "--vscode-input-placeholderForeground";
 export const VSC_BUTTON_BACKGROUND_VAR = "--vscode-button-background";
 export const VSC_BUTTON_FOREGROUND_VAR = "--vscode-button-foreground";
 export const VSC_EDITOR_BACKGROUND_VAR = "--vscode-editor-background";
@@ -27,6 +31,9 @@ export const VSC_THEME_COLOR_VARS = [
   VSC_INPUT_BACKGROUND_VAR,
   VSC_BACKGROUND_VAR,
   VSC_FOREGROUND_VAR,
+  VSC_FOREGROUND_MUTED_VAR,
+  VSC_DESCRIPTION_FOREGROUND,
+  VSC_INPUT_PLACEHOLDER_FOREGROUND,
   VSC_BUTTON_BACKGROUND_VAR,
   VSC_BUTTON_FOREGROUND_VAR,
   VSC_EDITOR_BACKGROUND_VAR,
@@ -51,6 +58,9 @@ export const vscInputBackground = `var(${VSC_INPUT_BACKGROUND_VAR}, rgb(45 45 45
 export const vscQuickInputBackground = `var(${VSC_QUICK_INPUT_BACKGROUND_VAR}, ${VSC_INPUT_BACKGROUND_VAR}, rgb(45 45 45))`;
 export const vscBackground = `var(${VSC_BACKGROUND_VAR}, rgb(30 30 30))`;
 export const vscForeground = `var(${VSC_FOREGROUND_VAR}, #fff)`;
+export const vscForegroundMuted = `var(${VSC_FOREGROUND_MUTED_VAR}, #999)`;
+export const vscDescriptionForeground = `var(${VSC_DESCRIPTION_FOREGROUND}, #999)`;
+export const vscInputPlaceholderForeground = `var(${VSC_INPUT_PLACEHOLDER_FOREGROUND}, #999)`;
 export const vscButtonBackground = `var(${VSC_BUTTON_BACKGROUND_VAR}, #1bbe84)`;
 export const vscButtonForeground = `var(${VSC_BUTTON_FOREGROUND_VAR}, #ffffff)`;
 export const vscEditorBackground = `var(${VSC_EDITOR_BACKGROUND_VAR}, ${VSC_BACKGROUND_VAR}, rgb(30 30 30))`;
diff --git a/gui/src/components/indexing/IndexingStatuses.tsx b/gui/src/components/indexing/IndexingStatuses.tsx
index 7441212f91..61bf102a98 100644
--- a/gui/src/components/indexing/IndexingStatuses.tsx
+++ b/gui/src/components/indexing/IndexingStatuses.tsx
@@ -34,20 +34,19 @@ function IndexingStatuses() {
         Manage your documentation sources
       </span>
       {/* <div className="flex max-h-[170px] flex-col gap-1 overflow-x-hidden overflow-y-scroll pr-2"> */}
-      {docsStatuses.length ? (
-        docsStatuses.map((status) => {
-          return <IndexingStatusViewer key={status.id} status={status} />;
-        })
-      ) : (
-        <SecondaryButton
-          onClick={() => {
-            dispatch(setShowDialog(true));
-            dispatch(setDialogMessage(<AddDocsDialog />));
-          }}
-        >
-          Add Docs
-        </SecondaryButton>
-      )}
+      {docsStatuses.length
+        ? docsStatuses.map((status) => {
+            return <IndexingStatusViewer key={status.id} status={status} />;
+          })
+        : null}
+      <SecondaryButton
+        onClick={() => {
+          dispatch(setShowDialog(true));
+          dispatch(setDialogMessage(<AddDocsDialog />));
+        }}
+      >
+        Add Docs
+      </SecondaryButton>
     </div>
     // </div>
   );
diff --git a/gui/tailwind.config.cjs b/gui/tailwind.config.cjs
index f021c748b8..3b3642d6bf 100644
--- a/gui/tailwind.config.cjs
+++ b/gui/tailwind.config.cjs
@@ -46,8 +46,15 @@ module.exports = {
         "vsc-badge-background": "var(--vscode-badge-background, #1bbe84)",
         "vsc-badge-foreground": "var(--vscode-badge-foreground, #fff)",
         "vsc-sidebar-border": "var(--vscode-sideBar-border, transparent)",
-        "vsc-find-match": "var(--vscode-editor-findMatchBackground, rgba(255, 255, 0, 0.6))",
-        "vsc-find-match-selected": "var(--vscode-editor-findMatchHighlightBackground, rgba(255, 223, 0, 0.8))",
+        "vsc-find-match":
+          "var(--vscode-editor-findMatchBackground, rgba(255, 255, 0, 0.6))",
+        "vsc-find-match-selected":
+          "var(--vscode-editor-findMatchHighlightBackground, rgba(255, 223, 0, 0.8))",
+        "vsc-foreground-muted": "var(--vscode-foreground-muted, #999)",
+        "vsc-description-foreground":
+          "var(--vscode-descriptionForeground, #999)",
+        "vsc-input-placeholder-foreground":
+          "var(--vscode-input-placeholderForeground, #999)",
       },
     },
   },

From 504c0c3843e5388fcd661f9755be9b174e9ef334 Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Fri, 22 Nov 2024 02:51:34 -0800
Subject: [PATCH 06/17] autodetect docs v1

---
 core/config/ProfileLifecycleManager.ts        |   2 -
 core/config/load.ts                           |   1 +
 core/core.ts                                  |   6 +
 core/index.d.ts                               |   2 +
 core/indexing/docs/suggestions/index.ts       |  39 +-
 .../suggestions/packageCrawlers/Python.ts     |   2 +-
 core/protocol/core.ts                         |   1 -
 core/protocol/webview.ts                      |   4 +-
 gui/src/components/dialogs/AddDocsDialog.tsx  | 469 ++++++++++--------
 gui/src/components/dialogs/index.tsx          |   2 +-
 .../components/indexing/ChatIndexingPeeks.tsx |   2 +-
 .../components/indexing/DocsIndexingPeeks.tsx |  76 +++
 .../components/indexing/IndexingStatuses.tsx  |  41 +-
 13 files changed, 391 insertions(+), 256 deletions(-)
 create mode 100644 gui/src/components/indexing/DocsIndexingPeeks.tsx

diff --git a/core/config/ProfileLifecycleManager.ts b/core/config/ProfileLifecycleManager.ts
index 85116f2133..8d44c8bac9 100644
--- a/core/config/ProfileLifecycleManager.ts
+++ b/core/config/ProfileLifecycleManager.ts
@@ -1,5 +1,3 @@
-import { config } from "dotenv";
-
 import {
   BrowserSerializedContinueConfig,
   ContinueConfig,
diff --git a/core/config/load.ts b/core/config/load.ts
index b692b36bf2..9911c840fe 100644
--- a/core/config/load.ts
+++ b/core/config/load.ts
@@ -527,6 +527,7 @@ function finalToBrowserConfig(
     embeddingsProvider: final.embeddingsProvider?.id,
     ui: final.ui,
     experimental: final.experimental,
+    docs: final.docs,
   };
 }
 
diff --git a/core/core.ts b/core/core.ts
index b2be3478c3..331ffbe857 100644
--- a/core/core.ts
+++ b/core/core.ts
@@ -735,6 +735,10 @@ export class Core {
       return this.docsService.initStatuses();
     });
     on("docs/getSuggestedDocs", async (msg) => {
+      if (hasRequestedDocs) {
+        return;
+      } // TODO, remove, hack because of rerendering
+      hasRequestedDocs = true;
       const suggestedDocs = await getAllSuggestedDocs(this.ide);
       this.messenger.send("docs/suggestions", suggestedDocs);
     });
@@ -842,3 +846,5 @@ export class Core {
 
   // private
 }
+
+let hasRequestedDocs = false;
diff --git a/core/index.d.ts b/core/index.d.ts
index 5541584ebd..35c072bdb2 100644
--- a/core/index.d.ts
+++ b/core/index.d.ts
@@ -1189,6 +1189,7 @@ export interface BrowserSerializedContinueConfig {
   reranker?: RerankerDescription;
   experimental?: ExperimentalConfig;
   analytics?: AnalyticsConfig;
+  docs?: SiteIndexingConfig[];
 }
 
 // DOCS SUGGESTIONS AND PACKAGE INFO
@@ -1210,6 +1211,7 @@ export type ParsedPackageInfo = {
 
 export type PackageDetails = {
   docsLink?: string;
+  docsLinkWarning?: string;
   title?: string;
   description?: string;
   repo?: string;
diff --git a/core/indexing/docs/suggestions/index.ts b/core/indexing/docs/suggestions/index.ts
index 4e0c2104ac..db192489ad 100644
--- a/core/indexing/docs/suggestions/index.ts
+++ b/core/indexing/docs/suggestions/index.ts
@@ -63,9 +63,10 @@ export async function getAllSuggestedDocs(ide: IDE) {
   );
 
   // Parse package files and build map of language -> packages
-  const packagesByRegistry: Record<string, ParsedPackageInfo[]> = {};
+  const packagesByCrawler: Record<string, ParsedPackageInfo[]> = {};
   PACKAGE_CRAWLERS.forEach((Crawler) => {
     const crawler = new Crawler();
+    packagesByCrawler[crawler.packageRegistry] = [];
     const packageFiles = packageFilesByRegistry[crawler.packageRegistry];
     packageFiles.forEach((file) => {
       const contents = fileContents.get(file.path);
@@ -73,10 +74,7 @@ export async function getAllSuggestedDocs(ide: IDE) {
         return;
       }
       const packages = crawler.parsePackageFile(file, contents);
-      if (!packagesByRegistry[crawler.packageRegistry]) {
-        packagesByRegistry[crawler.packageRegistry] = [];
-      }
-      packagesByRegistry[crawler.packageRegistry].push(...packages);
+      packagesByCrawler[crawler.packageRegistry].push(...packages);
     });
   });
 
@@ -84,13 +82,13 @@ export async function getAllSuggestedDocs(ide: IDE) {
   // TODO - this is where you would allow docs for different versions
   // by e.g. using "name-version" as the map key instead of just name
   // For now have not allowed
-  const languages = Object.keys(packagesByRegistry);
-  languages.forEach((language) => {
-    const packages = packagesByRegistry[language];
+  const registries = Object.keys(packagesByCrawler);
+  registries.forEach((registry) => {
+    const packages = packagesByCrawler[registry];
     const uniquePackages = Array.from(
       new Map(packages.map((pkg) => [pkg.name, pkg])).values(),
     );
-    packagesByRegistry[language] = uniquePackages;
+    packagesByCrawler[registry] = uniquePackages;
   });
 
   // Get documentation links for all packages
@@ -98,7 +96,7 @@ export async function getAllSuggestedDocs(ide: IDE) {
   await Promise.all(
     PACKAGE_CRAWLERS.map(async (Crawler) => {
       const crawler = new Crawler();
-      const packages = packagesByRegistry[crawler.packageRegistry];
+      const packages = packagesByCrawler[crawler.packageRegistry];
       const docsByRegistry = await Promise.all(
         packages.map(async (packageInfo) => {
           try {
@@ -114,6 +112,11 @@ export async function getAllSuggestedDocs(ide: IDE) {
               details: {
                 ...details,
                 docsLink: details.docsLink,
+                docsLinkWarning: details.docsLink.includes("github.com")
+                  ? "Github docs not supported, find the docs site"
+                  : details.docsLink.includes("docs")
+                    ? undefined
+                    : "May not be a docs site, check the URL",
               },
             };
           } catch (error) {
@@ -129,19 +132,3 @@ export async function getAllSuggestedDocs(ide: IDE) {
   );
   return allDocsResults;
 }
-
-// write me an interface PackageCrawler that contains:
-// 1. property `language` to store a given language like "python" or "typescript"
-// 2. has a method `getPackageFiles` which takes a list of file names and decides which ones match package/dependency files (e.g. package.json for typescript, requirements.txt for python, etc)
-// 3. has a method `parsePackageFile` which returns a list of package name and version from a relevant package file, in a standardized format like semver
-// 4. has a method `getDocumentationLink` to check for documentation link for a given package (e.g. GET `https://registry.npmjs.org/<package>` and find docs field for typescript, documentation link in the package metadata for PyPi, etc.)
-// Then, write typescript classes to implement this typescript interface for the languages "python" and "typescript"
-
-// I want to present the user with a list of dependencies and allow them to select which ones to index (embed) documentation for.
-// In order to prevent duplicate file reads, the process will be like this:
-// 1. take in a list of filepaths called `filepaths`
-// 2. loop an array of PackageCrawler classes to build a map of `language` (string) to `packageFilePaths` (string[])
-// 3. Get unique filepaths from `packageFilePaths` and build a map ` of filepath to file contents using an existing `readFile` function, and skipping file reads of already in the map
-// Finally,
-// Add a `` method to the interface and classes that returns
-// Then, assemble the classes in an array, and write a function getAllSuggestedDocs that returns a map of `language` to an ar
diff --git a/core/indexing/docs/suggestions/packageCrawlers/Python.ts b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
index d00045f976..6a486c9849 100644
--- a/core/indexing/docs/suggestions/packageCrawlers/Python.ts
+++ b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
@@ -30,7 +30,7 @@ export class PythonPackageCrawler implements PackageCrawler {
       .split("\n")
       .map((line) => {
         const [name, version] = line.split("==");
-        return { name, version, packageFile: file, language: "python" };
+        return { name, version, packageFile: file, language: "py" };
       })
       .filter((pkg) => pkg.name && pkg.version);
   }
diff --git a/core/protocol/core.ts b/core/protocol/core.ts
index 069f9e297a..f5b04c9cf5 100644
--- a/core/protocol/core.ts
+++ b/core/protocol/core.ts
@@ -9,7 +9,6 @@ import type {
   DiffLine,
   FileSymbolMap,
   IdeSettings,
-  IndexingStatusMap,
   LLMFullCompletionOptions,
   MessageContent,
   ModelDescription,
diff --git a/core/protocol/webview.ts b/core/protocol/webview.ts
index bf90497c8a..fb6a8837c5 100644
--- a/core/protocol/webview.ts
+++ b/core/protocol/webview.ts
@@ -2,9 +2,9 @@ import { ConfigValidationError } from "../config/validation.js";
 
 import type {
   ContextItemWithId,
-  DocsSuggestions,
   IndexingProgressUpdate,
   IndexingStatus,
+  PackageDocsResult,
 } from "../index.js";
 
 export type ToWebviewFromIdeOrCoreProtocol = {
@@ -26,5 +26,5 @@ export type ToWebviewFromIdeOrCoreProtocol = {
   getWebviewHistoryLength: [undefined, number];
   signInToControlPlane: [undefined, void];
   openDialogMessage: ["account", void];
-  "docs/suggestions": [DocsSuggestions, void];
+  "docs/suggestions": [PackageDocsResult[], void];
 };
diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx
index ca6a8bceb1..0a90a60d22 100644
--- a/gui/src/components/dialogs/AddDocsDialog.tsx
+++ b/gui/src/components/dialogs/AddDocsDialog.tsx
@@ -1,12 +1,15 @@
 import {
   CheckCircleIcon,
+  CheckIcon,
   ChevronDownIcon,
   ChevronUpIcon,
   CodeBracketIcon,
+  ExclamationTriangleIcon,
   InformationCircleIcon,
   LinkIcon,
+  PlusCircleIcon,
 } from "@heroicons/react/24/outline";
-import { PackageDocsResult, SiteIndexingConfig } from "core";
+import { IndexingStatus, PackageDocsResult, SiteIndexingConfig } from "core";
 import { usePostHog } from "posthog-js/react";
 import { useContext, useLayoutEffect, useMemo, useRef, useState } from "react";
 import { useDispatch, useSelector } from "react-redux";
@@ -20,39 +23,79 @@ import { RootState } from "../../redux/store";
 import IndexingStatusViewer from "../indexing/IndexingStatus";
 
 import { ToolTip } from "../gui/Tooltip";
+import FileIcon from "../FileIcon";
+import DocsIndexingPeeks from "../indexing/DocsIndexingPeeks";
+import { updateIndexingStatus } from "../../redux/slices/stateSlice";
+import preIndexedDocs from "core/indexing/docs/preIndexedDocs";
 
 function AddDocsDialog() {
   const posthog = usePostHog();
   const dispatch = useDispatch();
 
-  const ref = useRef<HTMLInputElement>(null);
+  const titleRef = useRef<HTMLInputElement>(null);
+  const urlRef = useRef<HTMLInputElement>(null);
 
   const [title, setTitle] = useState("");
   const [startUrl, setStartUrl] = useState("");
   const [faviconUrl, setFaviconUrl] = useState("");
-  const [isOpen, setIsOpen] = useState(false);
-
-  const [submittedConfig, setSubmittedConfig] = useState<SiteIndexingConfig>();
 
   const ideMessenger = useContext(IdeMessengerContext);
   const indexingStatuses = useSelector(
     (store: RootState) => store.state.indexing.statuses,
   );
 
+  const docsIndexingStatuses: IndexingStatus[] = useMemo(() => {
+    return Object.values(indexingStatuses).filter(
+      (status) => status.type === "docs" && status.status === "indexing",
+    );
+  }, [indexingStatuses]);
+
   const docsSuggestions = useSelector(
     (store: RootState) => store.state.docsSuggestions,
   );
 
+  const configDocs = useSelector((store: RootState) => store.state.config.docs);
+
+  const sortedDocsSuggestions = useMemo(() => {
+    const docsFromConfig = configDocs ?? [];
+    const filtered = docsSuggestions.filter((sug) => {
+      const startUrl = sug.details?.docsLink;
+      return (
+        !docsFromConfig.find((d) => d.startUrl === startUrl) &&
+        !docsIndexingStatuses.find((s) => s.id === startUrl) &&
+        (startUrl ? !preIndexedDocs[startUrl] : true)
+      );
+    });
+
+    filtered.sort((a, b) => {
+      const rank = (result: PackageDocsResult) => {
+        if (result.error) {
+          return 2;
+        } else if (result.details?.docsLinkWarning) {
+          return 1;
+        } else {
+          return 0;
+        }
+      };
+      return rank(a) - rank(b);
+    });
+    return filtered;
+  }, [docsSuggestions, configDocs, docsIndexingStatuses]);
+
   const isFormValid = startUrl && title;
 
   useLayoutEffect(() => {
     setTimeout(() => {
-      if (ref.current) {
-        ref.current.focus();
+      if (titleRef.current) {
+        titleRef.current.focus();
       }
     }, 100);
-  }, [ref]);
+  }, [titleRef]);
 
+  const closeDialog = () => {
+    dispatch(setShowDialog(false));
+    dispatch(setDialogMessage(undefined));
+  };
   function onSubmit(e: any) {
     e.preventDefault();
 
@@ -64,220 +107,236 @@ function AddDocsDialog() {
 
     ideMessenger.post("context/addDocs", siteIndexingConfig);
 
-    setSubmittedConfig(siteIndexingConfig);
     setTitle("");
     setStartUrl("");
     setFaviconUrl("");
 
     posthog.capture("add_docs_gui", { url: startUrl });
+
+    // Optimistic status update
+    dispatch(
+      updateIndexingStatus({
+        type: "docs",
+        description: "Initializing",
+        id: startUrl,
+        embeddingsProviderId: "mock-embeddings-provider-id",
+        progress: 0,
+        status: "indexing",
+        title,
+        url: startUrl,
+      }),
+    );
   }
 
-  const handleSelectSuggestion = (docsResult: PackageDocsResult) => {};
+  const handleSelectSuggestion = (docsResult: PackageDocsResult) => {
+    if (docsResult.error) {
+      setTitle(docsResult.packageInfo.name);
+      setStartUrl("");
+      urlRef.current?.focus();
+      return;
+    }
+    if (docsResult.details?.docsLinkWarning) {
+      setTitle(docsResult.packageInfo.name);
+      setStartUrl(docsResult.details.docsLink);
+      urlRef.current?.focus();
+      return;
+    }
+    const siteIndexingConfig: SiteIndexingConfig = {
+      startUrl: docsResult.details.docsLink,
+      title: docsResult.details.title,
+      faviconUrl: undefined,
+    };
+
+    ideMessenger.post("context/addDocs", siteIndexingConfig);
+
+    posthog.capture("add_docs_gui", { url: startUrl });
 
-  // if (submittedConfig) {
-  //   const status = indexingStatuses[submittedConfig.startUrl];
-  //   return (
-  //     <div className="flex flex-col p-4">
-  //       <div className="flex flex-row items-center gap-2">
-  //         <CheckCircleIcon className="h-8 w-8" />
-  //         <h1>{`Docs added`}</h1>
-  //       </div>
-  //       <div className="flex flex-col gap-1 text-stone-500">
-  //         <p className="m-0 p-0">Title: {submittedConfig.title}</p>
-  //         <p className="m-0 p-0">Start URL: {submittedConfig.startUrl}</p>
-  //         {submittedConfig.rootUrl && (
-  //           <p className="m-0 p-0">Root URL: {submittedConfig.rootUrl}</p>
-  //         )}
-  //         {submittedConfig.maxDepth && (
-  //           <p className="m-0 p-0">Max depth: {submittedConfig.maxDepth}</p>
-  //         )}
-  //         {submittedConfig.faviconUrl && (
-  //           <p className="m-0 p-0">Favicon URL: {submittedConfig.faviconUrl}</p>
-  //         )}
-  //       </div>
-  //       {!!status && (
-  //         <div className="mt-4 flex flex-col divide-x-0 divide-y-2 divide-solid divide-zinc-700">
-  //           <p className="m-0 mb-5 p-0 leading-snug">{`Type "@docs" and select ${submittedConfig.title} to reference these docs once indexing is complete. Check indexing status from the "More" page.`}</p>
-  //           <div className="pt-1">
-  //             <IndexingStatusViewer status={status} />
-  //           </div>
-  //         </div>
-  //       )}
-  //       <div className="mt-4 flex flex-row items-center justify-end gap-4">
-  //         <SecondaryButton
-  //           className=""
-  //           onClick={() => {
-  //             setSubmittedConfig(undefined);
-  //           }}
-  //         >
-  //           Add another
-  //         </SecondaryButton>
-  //         <Button
-  //           className=""
-  //           onClick={() => {
-  //             dispatch(setDialogMessage(undefined));
-  //             dispatch(setShowDialog(false));
-  //           }}
-  //         >
-  //           Done
-  //         </Button>
-  //       </div>
-  //     </div>
-  //   );
-  // }
+    // Optimistic status update
+    dispatch(
+      updateIndexingStatus({
+        type: "docs",
+        description: "Initializing",
+        id: docsResult.details.docsLink,
+        embeddingsProviderId: "mock-embeddings-provider-id",
+        progress: 0,
+        status: "indexing",
+        title: docsResult.details.title ?? docsResult.packageInfo.name,
+        url: docsResult.details.docsLink,
+      }),
+    );
+  };
 
   return (
-    <div className="p-4">
-      <div className="mb-8">
-        <h1>Add documentation</h1>
-        {docsSuggestions.length ? (
-          <div className="no-scrollbar max-h-[300px] overflow-y-auto">
-            <table className="border-collapse p-0">
-              <thead className="bg-vsc-background sticky -top-1 font-bold">
-                <tr className="">
-                  <td className="pr-1">
-                    <CodeBracketIcon className="h-3.5 w-3.5" />
-                  </td>
-                  <td className="pr-1">Title</td>
-                  {/* <td className="pr-1">Version</td> */}
-                  <td className="pr-1">Start Link</td>
-                  <td></td>
-                </tr>
-              </thead>
-              <tbody className="p-0">
-                <tr className="whitespace-nowrap">Add docs</tr>
-                {docsSuggestions.map((docsResult) => {
-                  const { error, details } = docsResult;
-                  const { language, name, version } = docsResult.packageInfo;
-                  const id = `${language}-${name}-${version}`;
-                  return (
-                    <tr key={id} className="p-0">
-                      <td>
-                        <input type="checkbox"></input>
-                      </td>
-                      <td>{name}</td>
-                      {/* <td>{version}</td> */}
-                      <td className="">
-                        {error ? (
-                          <span className="text-vsc-input-border italic">
-                            No docs link found
-                          </span>
-                        ) : (
-                          <span className="flex flex-row items-center gap-2">
-                            <div>
-                              <LinkIcon className="h-2 w-2" />
-                            </div>
-                            <p className="lines lines-1 m-0 p-0">
-                              {details.docsLink}
-                            </p>
-                          </span>
-                        )}
-                      </td>
-                      <td>
-                        <InformationCircleIcon
-                          data-tooltip-id={id}
-                          className="text-vsc-foreground-muted h-3.5 w-3.5 cursor-help"
+    <div className="px-2 py-4 sm:px-4">
+      <div className="">
+        <h1 className="mb-0 hidden sm:block">Add documentation</h1>
+        <h1 className="sm:hidden">Add docs</h1>
+        {sortedDocsSuggestions.length && (
+          <p className="m-0 mb-1 mt-4 p-0 font-semibold">Suggestions</p>
+        )}
+        <div className="border-vsc-foreground-muted max-h-[175px] overflow-y-scroll rounded-sm py-1 pr-2">
+          {sortedDocsSuggestions.map((docsResult) => {
+            const { error, details } = docsResult;
+            const { language, name, version } = docsResult.packageInfo;
+            const id = `${language}-${name}-${version}`;
+            return (
+              <>
+                <div
+                  key={id}
+                  className="grid cursor-pointer grid-cols-[auto_minmax(0,1fr)_minmax(0,1fr)_auto] items-center px-1 py-1 hover:bg-gray-200/10"
+                  onClick={() => {
+                    handleSelectSuggestion(docsResult);
+                  }}
+                >
+                  <div className="pr-1">
+                    {error ? (
+                      <div>
+                        <ExclamationTriangleIcon
+                          data-tooltip-id={id + "-error"}
+                          className="h-4 w-4 text-red-500"
                         />
-
-                        <ToolTip id={id} place="bottom">
-                          <p className="m-0 p-0">{`Version: ${version}`}</p>
-                          <p className="m-0 p-0">{`Found in ${docsResult.packageInfo.packageFile.path}`}</p>
+                        <ToolTip id={id + "-error"} place="bottom">
+                          Docs URL not found
                         </ToolTip>
-                      </td>
-                    </tr>
-                  );
-                })}
-              </tbody>
-            </table>
-          </div>
-        ) : null}
-        <p>
-          Continue pre-indexes many common documentation sites, but if there's
-          one you don't see in the dropdown, enter the URL here.
-        </p>
-
-        <p>
-          Continue's indexing engine will crawl the site and generate embeddings
-          so that you can ask questions.
-        </p>
-      </div>
-      {/* 
-      <form onSubmit={onSubmit} className="flex flex-col space-y-4">
-        <label>
-          Title
-          <Input
-            type="text"
-            placeholder="Title"
-            value={title}
-            ref={ref}
-            onChange={(e) => setTitle(e.target.value)}
-          />
-          <HelperText>
-            The title that will be displayed to users in the `@docs` submenu
-          </HelperText>
-        </label>
+                      </div>
+                    ) : details.docsLinkWarning ? (
+                      <div>
+                        <ExclamationTriangleIcon
+                          data-tooltip-id={id + "-warning"}
+                          className="h-4 w-4 text-yellow-600"
+                        />
+                        <ToolTip id={id + "-warning"} place="bottom">
+                          Start URL might not lead to docs
+                        </ToolTip>
+                      </div>
+                    ) : (
+                      <PlusCircleIcon className="h-4 w-4 text-green-600" />
+                    )}
+                  </div>
+                  <div className="flex items-center gap-0.5">
+                    <div className="hidden sm:block">
+                      <FileIcon
+                        filename={`x.${language}`}
+                        height="1rem"
+                        width="1rem"
+                      />
+                    </div>
+                    <span className="lines lines-1">{name}</span>
+                  </div>
+                  <div>
+                    {error ? (
+                      <span className="text-vsc-input-border italic">
+                        No docs link found
+                      </span>
+                    ) : (
+                      <div className="flex items-center gap-2">
+                        {/* <div>
+                        <LinkIcon className="h-2 w-2" />
+                      </div> */}
+                        <p className="lines lines-1 m-0 p-0">
+                          {details.docsLink}
+                        </p>
+                      </div>
+                    )}
+                  </div>
+                  <div>
+                    <InformationCircleIcon
+                      data-tooltip-id={id + "-info"}
+                      className="text-vsc-foreground-muted h-3.5 w-3.5 select-none"
+                    />
+                    <ToolTip id={id + "-info"} place="bottom">
+                      <p className="m-0 p-0">{`Version: ${version}`}</p>
+                      <p className="m-0 p-0">{`Found in ${docsResult.packageInfo.packageFile.path}`}</p>
+                    </ToolTip>
+                  </div>
+                </div>
+                <ToolTip id={id} place="bottom">
+                  {error ? "Add to form" : "Index these docs"}
+                </ToolTip>
+              </>
+            );
+          })}
+        </div>
+        <div className="mt-3">
+          <form onSubmit={onSubmit} className="flex flex-col gap-1">
+            <div className="flex flex-row gap-2">
+              <label className="flex min-w-16 basis-1/4 flex-col gap-1">
+                <div className="flex flex-row items-center gap-1">
+                  <span>Title</span>
+                  <div>
+                    <InformationCircleIcon
+                      data-tooltip-id={"add-docs-form-title"}
+                      className="text-vsc-foreground-muted h-3.5 w-3.5 select-none"
+                    />
+                    <ToolTip id={"add-docs-form-title"} place="top">
+                      The title that will be displayed to users in the `@docs`
+                      submenu
+                    </ToolTip>
+                  </div>
+                </div>
 
-        <label>
-          Start URL
-          <Input
-            type="url"
-            placeholder="Start URL"
-            value={startUrl}
-            onChange={(e) => {
-              setStartUrl(e.target.value);
-            }}
-          />
-          <HelperText>
-            The starting location to begin crawling the documentation site
-          </HelperText>
-        </label>
+                <Input
+                  type="text"
+                  placeholder="Title"
+                  value={title}
+                  ref={titleRef}
+                  onChange={(e) => setTitle(e.target.value)}
+                />
+              </label>
 
-        <div
-          className="cursor-pointer"
-          onClick={() => setIsOpen((prev) => !prev)}
-        >
-          {isOpen ? (
-            <ChevronUpIcon
-              width="1.0em"
-              height="1.0em"
-              style={{ color: lightGray }}
-            ></ChevronUpIcon>
-          ) : (
-            <ChevronDownIcon
-              width="1.0em"
-              height="1.0em"
-              style={{ color: lightGray }}
-            ></ChevronDownIcon>
-          )}
-          <span className="ms-1">Advanced</span>
+              <label className="flex basis-3/4 flex-col gap-1">
+                <div className="flex flex-row items-center gap-1">
+                  <span className="lines lines-1 whitespace-nowrap">
+                    Start URL
+                  </span>
+                  <div>
+                    <InformationCircleIcon
+                      data-tooltip-id={"add-docs-form-url"}
+                      className="text-vsc-foreground-muted h-3.5 w-3.5 select-none"
+                    />
+                    <ToolTip id={"add-docs-form-url"} place="top">
+                      The starting location to begin crawling the documentation
+                      site
+                    </ToolTip>
+                  </div>
+                </div>
+                <Input
+                  ref={urlRef}
+                  type="url"
+                  placeholder="Start URL"
+                  value={startUrl}
+                  onChange={(e) => {
+                    setStartUrl(e.target.value);
+                  }}
+                />
+              </label>
+            </div>
+            <div className="flex flex-row justify-end gap-2">
+              <SecondaryButton className="min-w-16" onClick={closeDialog}>
+                Done
+              </SecondaryButton>
+              <Button
+                className="min-w-16"
+                disabled={!isFormValid}
+                type="submit"
+              >
+                Go
+              </Button>
+            </div>
+          </form>
         </div>
+      </div>
 
-        {isOpen && (
-          <div className="pt-2">
-            <label>
-              Favicon URL [Optional]
-              <Input
-                type="url"
-                placeholder={`${startUrl}/favicon.ico`}
-                value={faviconUrl}
-                onChange={(e) => {
-                  setFaviconUrl(e.target.value);
-                }}
-              />
-              <HelperText>
-                The URL path to a favicon for the site - by default, it will be
-                `/favicon.ico` path from the Start URL
-              </HelperText>
-            </label>
-          </div>
-        )}
-
-        <div className="flex justify-end">
-          <Button disabled={!isFormValid} type="submit">
-            Submit
-          </Button>
+      <DocsIndexingPeeks statuses={docsIndexingStatuses} />
+      <div className="flex flex-row items-end justify-between gap-2">
+        <div>
+          {docsIndexingStatuses.length ? (
+            <p className="mt-2 p-0 text-xs text-stone-500">
+              It is safe to close this form while indexing
+            </p>
+          ) : null}
         </div>
-      </form> */}
+      </div>
     </div>
   );
 }
diff --git a/gui/src/components/dialogs/index.tsx b/gui/src/components/dialogs/index.tsx
index 7685109baf..8a3a67fe84 100644
--- a/gui/src/components/dialogs/index.tsx
+++ b/gui/src/components/dialogs/index.tsx
@@ -71,7 +71,7 @@ const TextDialog = (props: TextDialogProps) => {
   return (
     <ScreenCover onClick={props.onClose} hidden={!props.showDialog}>
       <DialogContainer
-        className="xs:w-[85%] w-[92%] sm:w-[75%]"
+        className="xs:w-[90%] no-scrollbar max-h-full w-[92%] max-w-[600px] overflow-auto sm:w-[88%] md:w-[80%]"
         onClick={(e) => {
           e.stopPropagation();
         }}
diff --git a/gui/src/components/indexing/ChatIndexingPeeks.tsx b/gui/src/components/indexing/ChatIndexingPeeks.tsx
index c77bdc323d..373689adc2 100644
--- a/gui/src/components/indexing/ChatIndexingPeeks.tsx
+++ b/gui/src/components/indexing/ChatIndexingPeeks.tsx
@@ -104,7 +104,7 @@ function ChatIndexingPeeks() {
   return (
     <div className="flex flex-col gap-1">
       {mergedIndexingStates.map((state) => {
-        return <ChatIndexingPeek state={state} />;
+        return <ChatIndexingPeek key={state.type} state={state} />;
       })}
     </div>
   );
diff --git a/gui/src/components/indexing/DocsIndexingPeeks.tsx b/gui/src/components/indexing/DocsIndexingPeeks.tsx
new file mode 100644
index 0000000000..e5b98d42e7
--- /dev/null
+++ b/gui/src/components/indexing/DocsIndexingPeeks.tsx
@@ -0,0 +1,76 @@
+import { useDispatch, useSelector } from "react-redux";
+import { RootState } from "../../redux/store";
+
+import { IndexingStatus } from "core";
+import { useMemo } from "react";
+import { useNavigate } from "react-router-dom";
+import { ArrowPathIcon, EyeSlashIcon } from "@heroicons/react/24/outline";
+import {
+  setDialogMessage,
+  setShowDialog,
+} from "../../redux/slices/uiStateSlice";
+
+export interface DocsIndexingPeekProps {
+  status: IndexingStatus;
+}
+
+function DocsIndexingPeek({ status }: DocsIndexingPeekProps) {
+  const navigate = useNavigate();
+  const dispatch = useDispatch();
+
+  const progressPercentage = useMemo(() => {
+    return Math.min(100, Math.max(0, status.progress * 100));
+  }, [status.progress]);
+
+  return (
+    <div
+      className="flex cursor-pointer flex-row items-center gap-2 rounded-md px-1 text-stone-500 hover:bg-gray-700/10"
+      onClick={() => {
+        navigate("/more");
+        dispatch(setShowDialog(false));
+        dispatch(setDialogMessage(undefined));
+      }}
+    >
+      <p className="m-0 p-0 text-stone-500 group-hover:underline">
+        {status.title}
+      </p>
+      <div className="my-2 h-1.5 flex-1 rounded-md border border-solid border-gray-400">
+        <div
+          className={`h-full rounded-lg bg-stone-500 transition-all duration-200 ease-in-out`}
+          style={{
+            width: `${progressPercentage}%`,
+          }}
+        />
+      </div>
+      <div className="xs:flex hidden flex-row items-center gap-1 text-stone-500">
+        <span className="text-xs no-underline">
+          {progressPercentage.toFixed(0)}%
+        </span>
+        <ArrowPathIcon
+          className={`animate-spin-slow inline-block h-4 w-4 text-stone-500`}
+        ></ArrowPathIcon>
+      </div>
+    </div>
+  );
+}
+
+interface DocsIndexingPeeksProps {
+  statuses: IndexingStatus[];
+}
+
+function DocsIndexingPeekList({ statuses }: DocsIndexingPeeksProps) {
+  if (!statuses.length) return null;
+
+  return (
+    <div className="flex flex-col">
+      <p className="mx-0 my-1.5 p-0 text-stone-500">Currently Indexing</p>
+      <div className="max-h-[100px] overflow-y-auto pr-2">
+        {statuses.map((status) => {
+          return <DocsIndexingPeek key={status.id} status={status} />;
+        })}
+      </div>
+    </div>
+  );
+}
+
+export default DocsIndexingPeekList;
diff --git a/gui/src/components/indexing/IndexingStatuses.tsx b/gui/src/components/indexing/IndexingStatuses.tsx
index 61bf102a98..746bc71ed9 100644
--- a/gui/src/components/indexing/IndexingStatuses.tsx
+++ b/gui/src/components/indexing/IndexingStatuses.tsx
@@ -8,6 +8,8 @@ import {
   setShowDialog,
 } from "../../redux/slices/uiStateSlice";
 import AddDocsDialog from "../dialogs/AddDocsDialog";
+import { PlusCircleIcon } from "@heroicons/react/24/outline";
+import { ToolTip } from "../gui/Tooltip";
 
 function IndexingStatuses() {
   const indexingStatuses = useSelector(
@@ -25,30 +27,35 @@ function IndexingStatuses() {
     <div className="flex flex-col gap-1">
       <div className="flex flex-row items-center justify-between">
         <h3 className="mb-1 mt-0 text-xl">@docs indexes</h3>
-        {/* <div className="border-1 rounded-full border">
-          <ChevronUpIcon className="h-8 w-8" />
-        </div> */}
-        {/* TODO add some way to hide, scroll, etc. */}
+        {docsStatuses.length ? (
+          <>
+            <PlusCircleIcon
+              data-tooltip-id={"more-add-docs-button"}
+              className="text-vsc-foreground-muted h-5 w-5 cursor-pointer focus:ring-0"
+              onClick={() => {
+                dispatch(setShowDialog(true));
+                dispatch(setDialogMessage(<AddDocsDialog />));
+              }}
+            />
+            <ToolTip id={"more-add-docs-button"} place="top">
+              Add Docs
+            </ToolTip>
+          </>
+        ) : null}
       </div>
       <span className="text-xs text-stone-500">
         Manage your documentation sources
       </span>
-      {/* <div className="flex max-h-[170px] flex-col gap-1 overflow-x-hidden overflow-y-scroll pr-2"> */}
-      {docsStatuses.length
-        ? docsStatuses.map((status) => {
+      <div className="flex max-h-[170px] flex-col gap-1 overflow-x-hidden overflow-y-scroll pr-2">
+        {docsStatuses.length ? (
+          docsStatuses.map((status) => {
             return <IndexingStatusViewer key={status.id} status={status} />;
           })
-        : null}
-      <SecondaryButton
-        onClick={() => {
-          dispatch(setShowDialog(true));
-          dispatch(setDialogMessage(<AddDocsDialog />));
-        }}
-      >
-        Add Docs
-      </SecondaryButton>
+        ) : (
+          <SecondaryButton>Add Docs</SecondaryButton>
+        )}
+      </div>
     </div>
-    // </div>
   );
 }
 

From b41e601798407615b504a2a52ab287d144f4716a Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Fri, 22 Nov 2024 03:05:58 -0800
Subject: [PATCH 07/17] suggested docs UI tweaks

---
 .../suggestions/packageCrawlers/Python.ts     |  1 -
 extensions/vscode/package-lock.json           |  2 +-
 gui/package-lock.json                         |  8 +------
 gui/src/components/dialogs/AddDocsDialog.tsx  | 24 ++++++++++---------
 .../components/indexing/DocsIndexingPeeks.tsx |  9 +++----
 5 files changed, 20 insertions(+), 24 deletions(-)

diff --git a/core/indexing/docs/suggestions/packageCrawlers/Python.ts b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
index 6a486c9849..3a617568f4 100644
--- a/core/indexing/docs/suggestions/packageCrawlers/Python.ts
+++ b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
@@ -39,7 +39,6 @@ export class PythonPackageCrawler implements PackageCrawler {
     packageInfo: ParsedPackageInfo,
   ): Promise<PackageDetails> {
     // Fetch metadata from PyPI to find the documentation link
-
     const response = await fetch(
       `https://pypi.org/pypi/${packageInfo.name}/json`,
     );
diff --git a/extensions/vscode/package-lock.json b/extensions/vscode/package-lock.json
index b30a951dfc..8d680fb94a 100644
--- a/extensions/vscode/package-lock.json
+++ b/extensions/vscode/package-lock.json
@@ -109,7 +109,7 @@
         "cheerio": "^1.0.0-rc.12",
         "commander": "^12.0.0",
         "comment-json": "^4.2.3",
-        "dbinfoz": "^0.11.0",
+        "dbinfoz": "^0.14.0",
         "diff": "^7.0.0",
         "dotenv": "^16.4.5",
         "fastest-levenshtein": "^1.0.16",
diff --git a/gui/package-lock.json b/gui/package-lock.json
index 3be234fb8d..1a9e4fe3ff 100644
--- a/gui/package-lock.json
+++ b/gui/package-lock.json
@@ -56,7 +56,6 @@
         "socket.io-client": "^4.7.2",
         "styled-components": "^5.3.6",
         "table": "^6.8.1",
-        "tailwind-scrollbar-hide": "^1.1.7",
         "tippy.js": "^6.3.7",
         "unist-util-visit": "^5.0.0",
         "uuid": "^9.0.1",
@@ -112,7 +111,7 @@
         "cheerio": "^1.0.0-rc.12",
         "commander": "^12.0.0",
         "comment-json": "^4.2.3",
-        "dbinfoz": "^0.11.0",
+        "dbinfoz": "^0.14.0",
         "diff": "^7.0.0",
         "dotenv": "^16.4.5",
         "fastest-levenshtein": "^1.0.16",
@@ -8984,11 +8983,6 @@
         "node": ">=10.0.0"
       }
     },
-    "node_modules/tailwind-scrollbar-hide": {
-      "version": "1.1.7",
-      "resolved": "https://registry.npmjs.org/tailwind-scrollbar-hide/-/tailwind-scrollbar-hide-1.1.7.tgz",
-      "integrity": "sha512-X324n9OtpTmOMqEgDUEA/RgLrNfBF/jwJdctaPZDzB3mppxJk7TLIDmOreEDm1Bq4R9LSPu4Epf8VSdovNU+iA=="
-    },
     "node_modules/tailwindcss": {
       "version": "3.4.12",
       "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.12.tgz",
diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx
index 0a90a60d22..1467a3fa50 100644
--- a/gui/src/components/dialogs/AddDocsDialog.tsx
+++ b/gui/src/components/dialogs/AddDocsDialog.tsx
@@ -167,14 +167,17 @@ function AddDocsDialog() {
   };
 
   return (
-    <div className="px-2 py-4 sm:px-4">
-      <div className="">
+    <div className="px-2 pt-4 sm:px-4">
+      <div className="mb-2">
         <h1 className="mb-0 hidden sm:block">Add documentation</h1>
         <h1 className="sm:hidden">Add docs</h1>
+        <p className="m-0 mt-2 p-0 text-stone-500">
+          For the @docs context provider
+        </p>
         {sortedDocsSuggestions.length && (
           <p className="m-0 mb-1 mt-4 p-0 font-semibold">Suggestions</p>
         )}
-        <div className="border-vsc-foreground-muted max-h-[175px] overflow-y-scroll rounded-sm py-1 pr-2">
+        <div className="border-vsc-foreground-muted max-h-[145px] overflow-y-scroll rounded-sm py-1 pr-2">
           {sortedDocsSuggestions.map((docsResult) => {
             const { error, details } = docsResult;
             const { language, name, version } = docsResult.packageInfo;
@@ -328,14 +331,13 @@ function AddDocsDialog() {
       </div>
 
       <DocsIndexingPeeks statuses={docsIndexingStatuses} />
-      <div className="flex flex-row items-end justify-between gap-2">
-        <div>
-          {docsIndexingStatuses.length ? (
-            <p className="mt-2 p-0 text-xs text-stone-500">
-              It is safe to close this form while indexing
-            </p>
-          ) : null}
-        </div>
+      <div className="flex flex-row items-end justify-start gap-2">
+        {docsIndexingStatuses.length ? (
+          <p className="mt-2 flex flex-row items-center gap-1 p-0 px-1 text-xs text-stone-500">
+            <CheckIcon className="h-3 w-3" />
+            It is safe to close this form while indexing
+          </p>
+        ) : null}
       </div>
     </div>
   );
diff --git a/gui/src/components/indexing/DocsIndexingPeeks.tsx b/gui/src/components/indexing/DocsIndexingPeeks.tsx
index e5b98d42e7..9c5f96ec85 100644
--- a/gui/src/components/indexing/DocsIndexingPeeks.tsx
+++ b/gui/src/components/indexing/DocsIndexingPeeks.tsx
@@ -1,10 +1,9 @@
-import { useDispatch, useSelector } from "react-redux";
-import { RootState } from "../../redux/store";
+import { useDispatch } from "react-redux";
 
 import { IndexingStatus } from "core";
 import { useMemo } from "react";
 import { useNavigate } from "react-router-dom";
-import { ArrowPathIcon, EyeSlashIcon } from "@heroicons/react/24/outline";
+import { ArrowPathIcon } from "@heroicons/react/24/outline";
 import {
   setDialogMessage,
   setShowDialog,
@@ -63,7 +62,9 @@ function DocsIndexingPeekList({ statuses }: DocsIndexingPeeksProps) {
 
   return (
     <div className="flex flex-col">
-      <p className="mx-0 my-1.5 p-0 text-stone-500">Currently Indexing</p>
+      <p className="mx-0 my-1.5 p-0 px-1 font-semibold text-stone-500">
+        Currently Indexing
+      </p>
       <div className="max-h-[100px] overflow-y-auto pr-2">
         {statuses.map((status) => {
           return <DocsIndexingPeek key={status.id} status={status} />;

From 6cb9adc110e774e4686f99c1fbe7b1de2e438878 Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Fri, 22 Nov 2024 03:10:45 -0800
Subject: [PATCH 08/17] suggested docs - cleanup

---
 gui/src/components/dialogs/SuggestedDocs.tsx |  18 ----
 manual-testing-sandbox/package.json          | 102 -------------------
 manual-testing-sandbox/requirements.txt      |  10 --
 3 files changed, 130 deletions(-)
 delete mode 100644 gui/src/components/dialogs/SuggestedDocs.tsx
 delete mode 100644 manual-testing-sandbox/package.json
 delete mode 100644 manual-testing-sandbox/requirements.txt

diff --git a/gui/src/components/dialogs/SuggestedDocs.tsx b/gui/src/components/dialogs/SuggestedDocs.tsx
deleted file mode 100644
index 1540ea92f9..0000000000
--- a/gui/src/components/dialogs/SuggestedDocs.tsx
+++ /dev/null
@@ -1,18 +0,0 @@
-import { PackageDocsResult } from "core";
-
-interface SuggestedDocProps {
-  docResult: PackageDocsResult;
-}
-const SuggestedDoc = ({ docResult }: SuggestedDocProps) => {
-  return <div>SuggestedDoc</div>;
-};
-
-interface SuggestedDocsListProps {
-  docs: PackageDocsResult[];
-  //   on
-}
-const SuggestedDocsList = ({ docs }: SuggestedDocsListProps) => {
-  return <div></div>;
-};
-
-export default SuggestedDocsList;
diff --git a/manual-testing-sandbox/package.json b/manual-testing-sandbox/package.json
deleted file mode 100644
index ef77ac910c..0000000000
--- a/manual-testing-sandbox/package.json
+++ /dev/null
@@ -1,102 +0,0 @@
-{
-  "name": "gui",
-  "private": true,
-  "type": "module",
-  "author": "Continue Dev, Inc",
-  "license": "Apache-2.0",
-  "scripts": {
-    "dev": "vite",
-    "tsc:check": "tsc -p ./ --noEmit",
-    "build": "tsc && vite build",
-    "preview": "vite preview",
-    "test": "vitest run",
-    "test:coverage": "vitest run --coverage",
-    "test:ui": "vitest --ui",
-    "test:watch": "vitest"
-  },
-  "dependencies": {
-    "@headlessui/react": "^1.7.17",
-    "@heroicons/react": "^2.0.18",
-    "@monaco-editor/react": "^4.6.0",
-    "@reduxjs/toolkit": "^1.9.3",
-    "@tiptap/core": "^2.3.2",
-    "@tiptap/extension-document": "^2.3.2",
-    "@tiptap/extension-dropcursor": "^2.1.16",
-    "@tiptap/extension-history": "^2.3.2",
-    "@tiptap/extension-image": "^2.1.16",
-    "@tiptap/extension-mention": "^2.1.13",
-    "@tiptap/extension-paragraph": "^2.3.2",
-    "@tiptap/extension-placeholder": "^2.1.13",
-    "@tiptap/extension-text": "^2.3.2",
-    "@tiptap/pm": "^2.1.13",
-    "@tiptap/react": "^2.1.13",
-    "@tiptap/starter-kit": "^2.1.13",
-    "@tiptap/suggestion": "^2.1.13",
-    "@types/vscode-webview": "^1.57.1",
-    "core": "file:../core",
-    "dompurify": "^3.0.6",
-    "downshift": "^7.6.0",
-    "lodash": "^4.17.21",
-    "minisearch": "^7.0.2",
-    "onigasm": "^2.2.5",
-    "posthog-js": "^1.130.1",
-    "prismjs": "^1.29.0",
-    "react": "^18.2.0",
-    "react-dom": "^18.2.0",
-    "react-error-boundary": "^4.0.11",
-    "react-hook-form": "^7.47.0",
-    "react-intersection-observer": "^9.13.1",
-    "react-markdown": "^9.0.1",
-    "react-redux": "^8.0.5",
-    "react-remark": "^2.1.0",
-    "react-router-dom": "^6.14.2",
-    "react-switch": "^7.0.0",
-    "react-syntax-highlighter": "^15.5.0",
-    "react-tooltip": "^5.18.0",
-    "redux-persist": "^6.0.0",
-    "redux-persist-transform-filter": "^0.0.22",
-    "rehype-highlight": "^7.0.0",
-    "rehype-katex": "^7.0.0",
-    "rehype-wrap-all": "^1.1.0",
-    "remark-math": "^6.0.0",
-    "reselect": "^5.1.1",
-    "seti-file-icons": "^0.0.8",
-    "socket.io-client": "^4.7.2",
-    "styled-components": "^5.3.6",
-    "table": "^6.8.1",
-    "tippy.js": "^6.3.7",
-    "unist-util-visit": "^5.0.0",
-    "uuid": "^9.0.1",
-    "vscode-webview": "^1.0.1-beta.1"
-  },
-  "devDependencies": {
-    "@swc/cli": "^0.3.14",
-    "@swc/core": "^1.7.26",
-    "@testing-library/dom": "^10.4.0",
-    "@testing-library/jest-dom": "^6.5.0",
-    "@testing-library/react": "^16.0.1",
-    "@testing-library/user-event": "^14.5.2",
-    "@types/lodash": "^4.17.6",
-    "@types/node": "^20.5.6",
-    "@types/node-fetch": "^2.6.4",
-    "@types/react": "^18.3.11",
-    "@types/react-dom": "^18.3.1",
-    "@types/react-router-dom": "^5.3.3",
-    "@types/react-syntax-highlighter": "^15.5.7",
-    "@types/styled-components": "^5.1.26",
-    "@vitejs/plugin-react-swc": "^3.7.0",
-    "@vitest/coverage-v8": "^2.1.3",
-    "@vitest/ui": "^2.1.3",
-    "autoprefixer": "^10.4.13",
-    "jsdom": "^25.0.1",
-    "postcss": "^8.4.21",
-    "tailwindcss": "^3.2.7",
-    "typescript": "^4.9.3",
-    "vite": "^4.1.0",
-    "vitest": "^2.1.3"
-  },
-  "engine-strict": true,
-  "engines": {
-    "node": ">=20.11.0"
-  }
-}
diff --git a/manual-testing-sandbox/requirements.txt b/manual-testing-sandbox/requirements.txt
deleted file mode 100644
index 793d6a8275..0000000000
--- a/manual-testing-sandbox/requirements.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-flask==2.1.1
-requests==2.28.1
-numpy==1.23.4
-pandas==1.5.0
-scipy==1.9.3
-django==4.1.3
-matplotlib==3.6.2
-pytest==7.2.0
-sqlalchemy==1.4.41
-beautifulsoup4==4.11.1
\ No newline at end of file

From 0b0a0e644525f30f7617b95da0347e8042b64f39 Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Fri, 22 Nov 2024 11:07:28 -0800
Subject: [PATCH 09/17] docs suggestions patrick bug fixes p1

---
 manual-testing-sandbox/package.json     | 45 +++++++++++++++++++
 manual-testing-sandbox/requirements.txt | 57 +++++++++++++++++++++++++
 2 files changed, 102 insertions(+)
 create mode 100644 manual-testing-sandbox/package.json
 create mode 100644 manual-testing-sandbox/requirements.txt

diff --git a/manual-testing-sandbox/package.json b/manual-testing-sandbox/package.json
new file mode 100644
index 0000000000..5bcec0c8fc
--- /dev/null
+++ b/manual-testing-sandbox/package.json
@@ -0,0 +1,45 @@
+{
+  "name": "my-react-project",
+  "version": "1.0.0",
+  "description": "A React project using Radix UI for accessible components",
+  "main": "index.js",
+  "scripts": {
+    "start": "react-scripts start",
+    "build": "react-scripts build",
+    "test": "react-scripts test",
+    "eject": "react-scripts eject"
+  },
+  "keywords": [
+    "react",
+    "radix-ui",
+    "accessibility",
+    "frontend"
+  ],
+  "author": "Your Name",
+  "license": "MIT",
+  "dependencies": {
+    "@radix-ui/react-accordion": "^1.0.0",
+    "@radix-ui/react-alert-dialog": "^1.0.0",
+    "@radix-ui/react-checkbox": "^1.0.0",
+    "react": "^18.0.0",
+    "react-dom": "^18.0.0",
+    "react-scripts": "^5.0.0"
+  },
+  "devDependencies": {
+    "@types/react": "^18.0.0",
+    "@types/react-dom": "^18.0.0",
+    "typescript": "^4.5.0"
+  },
+  "browserslist": {
+    "production": [
+      ">0.2%",
+      "not dead",
+      "not op_mini all"
+    ],
+    "development": [
+      "last 1 chrome version",
+      "last 1 firefox version",
+      "last 1 safari version"
+    ]
+  }
+}
diff --git a/manual-testing-sandbox/requirements.txt b/manual-testing-sandbox/requirements.txt
new file mode 100644
index 0000000000..87f2e654bf
--- /dev/null
+++ b/manual-testing-sandbox/requirements.txt
@@ -0,0 +1,57 @@
+# Core packages
+numpy==1.21.2
+pandas==1.3.3
+scipy==1.7.1
+
+# Web development
+flask==2.0.2
+django==3.2.7
+
+# Request handling
+requests==2.26.0
+httpx==0.19.0
+
+# Machine Learning
+scikit-learn==0.24.2
+tensorflow==2.6.0
+
+# Data visualization
+matplotlib==3.4.3
+seaborn==0.11.2
+plotly==5.3.1
+
+# Natural Language Processing
+nltk==3.6.3
+spacy==3.1.2
+
+# Deep Learning
+torch==1.9.1
+keras==2.6.0
+
+# Image processing
+pillow==8.3.2
+opencv-python==4.5.3.56
+
+# Data handling and manipulation
+beautifulsoup4==4.10.0
+sqlalchemy==1.4.25
+xlrd==2.0.1
+openpyxl==3.0.9
+
+# Utilities
+pyyaml==5.4.1
+python-dotenv==0.19.0
+
+# Testing
+pytest==6.2.5
+
+# Logging and Debugging
+loguru==0.5.3
+
+# Asynchronous Programming
+aiohttp==3.7.4
+
+# Others
+jupyter==1.0.0
+gunicorn==20.1.0
+psycopg2==2.9.1
\ No newline at end of file

From f4e9da5828b9f6040728f570692ddfd4f5634e55 Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Fri, 22 Nov 2024 12:19:33 -0800
Subject: [PATCH 10/17] patrick tweaks

---
 .../docs/suggestions/packageCrawlers/TsJs.ts  |   5 +
 gui/src/components/AccountDialog.tsx          |   2 -
 gui/src/components/dialogs/AddDocsDialog.tsx  | 159 +++++++++---------
 .../components/indexing/DocsIndexingPeeks.tsx |  20 ++-
 .../components/indexing/IndexingStatuses.tsx  |  37 ++--
 5 files changed, 115 insertions(+), 108 deletions(-)

diff --git a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
index 42c5007480..3a9ef96c2b 100644
--- a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
+++ b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
@@ -59,6 +59,11 @@ export class NodePackageCrawler implements PackageCrawler {
       throw new Error(`Could not fetch data for package ${name}`);
     }
     const data = await response.json();
+
+    //   const dependencies = Object.keys(packageContentData.dependencies || {})
+    //   .concat(Object.keys(packageContentData.devDependencies || {}));
+    // const usesTypescript = dependencies.includes("typescript");
+
     return {
       docsLink: data.homepage as string | undefined,
       title: name, // package.json doesn't have specific title field
diff --git a/gui/src/components/AccountDialog.tsx b/gui/src/components/AccountDialog.tsx
index 5438d11df6..db61fe3c0e 100644
--- a/gui/src/components/AccountDialog.tsx
+++ b/gui/src/components/AccountDialog.tsx
@@ -65,8 +65,6 @@ function AccountDialog() {
     );
   }
 
-  const dispatch = useDispatch();
-
   const changeProfileId = (id: string) => {
     ideMessenger.post("didChangeSelectedProfile", { id });
     dispatch(setSelectedProfileId(id));
diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx
index 1467a3fa50..68039e85c3 100644
--- a/gui/src/components/dialogs/AddDocsDialog.tsx
+++ b/gui/src/components/dialogs/AddDocsDialog.tsx
@@ -7,7 +7,9 @@ import {
   ExclamationTriangleIcon,
   InformationCircleIcon,
   LinkIcon,
+  PencilIcon,
   PlusCircleIcon,
+  PlusIcon,
 } from "@heroicons/react/24/outline";
 import { IndexingStatus, PackageDocsResult, SiteIndexingConfig } from "core";
 import { usePostHog } from "posthog-js/react";
@@ -27,6 +29,7 @@ import FileIcon from "../FileIcon";
 import DocsIndexingPeeks from "../indexing/DocsIndexingPeeks";
 import { updateIndexingStatus } from "../../redux/slices/stateSlice";
 import preIndexedDocs from "core/indexing/docs/preIndexedDocs";
+import { NewSessionButton } from "../mainInput/NewSessionButton";
 
 function AddDocsDialog() {
   const posthog = usePostHog();
@@ -168,13 +171,13 @@ function AddDocsDialog() {
 
   return (
     <div className="px-2 pt-4 sm:px-4">
-      <div className="mb-2">
+      <div className="">
         <h1 className="mb-0 hidden sm:block">Add documentation</h1>
         <h1 className="sm:hidden">Add docs</h1>
         <p className="m-0 mt-2 p-0 text-stone-500">
           For the @docs context provider
         </p>
-        {sortedDocsSuggestions.length && (
+        {!!sortedDocsSuggestions.length && (
           <p className="m-0 mb-1 mt-4 p-0 font-semibold">Suggestions</p>
         )}
         <div className="border-vsc-foreground-muted max-h-[145px] overflow-y-scroll rounded-sm py-1 pr-2">
@@ -183,80 +186,65 @@ function AddDocsDialog() {
             const { language, name, version } = docsResult.packageInfo;
             const id = `${language}-${name}-${version}`;
             return (
-              <>
-                <div
-                  key={id}
-                  className="grid cursor-pointer grid-cols-[auto_minmax(0,1fr)_minmax(0,1fr)_auto] items-center px-1 py-1 hover:bg-gray-200/10"
-                  onClick={() => {
-                    handleSelectSuggestion(docsResult);
-                  }}
-                >
-                  <div className="pr-1">
-                    {error ? (
-                      <div>
-                        <ExclamationTriangleIcon
-                          data-tooltip-id={id + "-error"}
-                          className="h-4 w-4 text-red-500"
-                        />
-                        <ToolTip id={id + "-error"} place="bottom">
-                          Docs URL not found
-                        </ToolTip>
-                      </div>
-                    ) : details.docsLinkWarning ? (
-                      <div>
-                        <ExclamationTriangleIcon
-                          data-tooltip-id={id + "-warning"}
-                          className="h-4 w-4 text-yellow-600"
-                        />
-                        <ToolTip id={id + "-warning"} place="bottom">
-                          Start URL might not lead to docs
-                        </ToolTip>
-                      </div>
-                    ) : (
-                      <PlusCircleIcon className="h-4 w-4 text-green-600" />
-                    )}
-                  </div>
-                  <div className="flex items-center gap-0.5">
-                    <div className="hidden sm:block">
-                      <FileIcon
-                        filename={`x.${language}`}
-                        height="1rem"
-                        width="1rem"
+              <div
+                key={id}
+                className="grid cursor-pointer grid-cols-[auto_minmax(0,1fr)_minmax(0,1fr)_auto] items-center px-1 py-1 hover:bg-gray-200/10"
+                onClick={() => {
+                  handleSelectSuggestion(docsResult);
+                }}
+              >
+                <div className="pr-1">
+                  {error || details.docsLinkWarning ? (
+                    <div>
+                      <PencilIcon
+                        data-tooltip-id={id + "-edit"}
+                        className="vsc-foreground-muted h-3 w-3"
                       />
+                      <ToolTip id={id + "-edit"} place="bottom">
+                        This may not be a docs page
+                      </ToolTip>
                     </div>
-                    <span className="lines lines-1">{name}</span>
+                  ) : (
+                    <PlusIcon className="text-foreground-muted h-3.5 w-3.5" />
+                  )}
+                </div>
+                <div className="flex items-center gap-0.5">
+                  <div className="hidden sm:block">
+                    <FileIcon
+                      filename={`x.${language}`}
+                      height="1rem"
+                      width="1rem"
+                    />
                   </div>
-                  <div>
-                    {error ? (
-                      <span className="text-vsc-input-border italic">
-                        No docs link found
-                      </span>
-                    ) : (
-                      <div className="flex items-center gap-2">
-                        {/* <div>
+                  <span className="lines lines-1">{name}</span>
+                </div>
+                <div>
+                  {error ? (
+                    <span className="text-vsc-foreground-muted italic">
+                      No docs link found
+                    </span>
+                  ) : (
+                    <div className="flex items-center gap-2">
+                      {/* <div>
                         <LinkIcon className="h-2 w-2" />
                       </div> */}
-                        <p className="lines lines-1 m-0 p-0">
-                          {details.docsLink}
-                        </p>
-                      </div>
-                    )}
-                  </div>
-                  <div>
-                    <InformationCircleIcon
-                      data-tooltip-id={id + "-info"}
-                      className="text-vsc-foreground-muted h-3.5 w-3.5 select-none"
-                    />
-                    <ToolTip id={id + "-info"} place="bottom">
-                      <p className="m-0 p-0">{`Version: ${version}`}</p>
-                      <p className="m-0 p-0">{`Found in ${docsResult.packageInfo.packageFile.path}`}</p>
-                    </ToolTip>
-                  </div>
+                      <p className="lines lines-1 m-0 p-0">
+                        {details.docsLink}
+                      </p>
+                    </div>
+                  )}
                 </div>
-                <ToolTip id={id} place="bottom">
-                  {error ? "Add to form" : "Index these docs"}
-                </ToolTip>
-              </>
+                <div>
+                  <InformationCircleIcon
+                    data-tooltip-id={id + "-info"}
+                    className="text-vsc-foreground-muted h-3.5 w-3.5 select-none"
+                  />
+                  <ToolTip id={id + "-info"} place="bottom">
+                    <p className="m-0 p-0">{`Version: ${version}`}</p>
+                    <p className="m-0 p-0">{`Found in ${docsResult.packageInfo.packageFile.path}`}</p>
+                  </ToolTip>
+                </div>
+              </div>
             );
           })}
         </div>
@@ -313,31 +301,36 @@ function AddDocsDialog() {
                   }}
                 />
               </label>
+              {/* <div>
+                <PlusCircleIcon className="h-5 w-5 self-end" />
+              </div> */}
             </div>
             <div className="flex flex-row justify-end gap-2">
-              <SecondaryButton className="min-w-16" onClick={closeDialog}>
-                Done
-              </SecondaryButton>
-              <Button
+              <SecondaryButton
                 className="min-w-16"
                 disabled={!isFormValid}
                 type="submit"
               >
-                Go
-              </Button>
+                Add
+              </SecondaryButton>
             </div>
           </form>
         </div>
       </div>
 
       <DocsIndexingPeeks statuses={docsIndexingStatuses} />
-      <div className="flex flex-row items-end justify-start gap-2">
-        {docsIndexingStatuses.length ? (
-          <p className="mt-2 flex flex-row items-center gap-1 p-0 px-1 text-xs text-stone-500">
-            <CheckIcon className="h-3 w-3" />
-            It is safe to close this form while indexing
-          </p>
-        ) : null}
+      <div className="flex flex-row items-end justify-between pb-3">
+        <div>
+          {docsIndexingStatuses.length ? (
+            <p className="mt-2 flex flex-row items-center gap-1 p-0 px-1 text-xs text-stone-500">
+              <CheckIcon className="h-3 w-3" />
+              It is safe to close this form while indexing
+            </p>
+          ) : null}
+        </div>
+        {/* <Button className="min-w-16" onClick={closeDialog}>
+          Done
+        </Button> */}
       </div>
     </div>
   );
diff --git a/gui/src/components/indexing/DocsIndexingPeeks.tsx b/gui/src/components/indexing/DocsIndexingPeeks.tsx
index 9c5f96ec85..e50b4adcf2 100644
--- a/gui/src/components/indexing/DocsIndexingPeeks.tsx
+++ b/gui/src/components/indexing/DocsIndexingPeeks.tsx
@@ -45,9 +45,9 @@ function DocsIndexingPeek({ status }: DocsIndexingPeekProps) {
         <span className="text-xs no-underline">
           {progressPercentage.toFixed(0)}%
         </span>
-        <ArrowPathIcon
+        {/* <ArrowPathIcon
           className={`animate-spin-slow inline-block h-4 w-4 text-stone-500`}
-        ></ArrowPathIcon>
+        ></ArrowPathIcon> */}
       </div>
     </div>
   );
@@ -58,17 +58,19 @@ interface DocsIndexingPeeksProps {
 }
 
 function DocsIndexingPeekList({ statuses }: DocsIndexingPeeksProps) {
-  if (!statuses.length) return null;
-
   return (
-    <div className="flex flex-col">
+    <div className="border-vsc-input-border mt-2 flex flex-col border-0 border-t border-solid pt-2">
       <p className="mx-0 my-1.5 p-0 px-1 font-semibold text-stone-500">
-        Currently Indexing
+        Currently Indexing:
       </p>
       <div className="max-h-[100px] overflow-y-auto pr-2">
-        {statuses.map((status) => {
-          return <DocsIndexingPeek key={status.id} status={status} />;
-        })}
+        {statuses.length ? (
+          statuses.map((status) => {
+            return <DocsIndexingPeek key={status.id} status={status} />;
+          })
+        ) : (
+          <p className="m-0 pl-1 font-semibold text-stone-500">None</p>
+        )}
       </div>
     </div>
   );
diff --git a/gui/src/components/indexing/IndexingStatuses.tsx b/gui/src/components/indexing/IndexingStatuses.tsx
index 746bc71ed9..3ea04216f4 100644
--- a/gui/src/components/indexing/IndexingStatuses.tsx
+++ b/gui/src/components/indexing/IndexingStatuses.tsx
@@ -28,20 +28,29 @@ function IndexingStatuses() {
       <div className="flex flex-row items-center justify-between">
         <h3 className="mb-1 mt-0 text-xl">@docs indexes</h3>
         {docsStatuses.length ? (
-          <>
-            <PlusCircleIcon
-              data-tooltip-id={"more-add-docs-button"}
-              className="text-vsc-foreground-muted h-5 w-5 cursor-pointer focus:ring-0"
-              onClick={() => {
-                dispatch(setShowDialog(true));
-                dispatch(setDialogMessage(<AddDocsDialog />));
-              }}
-            />
-            <ToolTip id={"more-add-docs-button"} place="top">
-              Add Docs
-            </ToolTip>
-          </>
-        ) : null}
+          <div
+            className="border-vsc-foreground text-vsc-foreground enabled:hover:bg-vsc-background m-2 rounded border border-solid bg-inherit px-3 py-2 enabled:hover:cursor-pointer enabled:hover:opacity-90 disabled:text-gray-500"
+            onClick={() => {
+              dispatch(setShowDialog(true));
+              dispatch(setDialogMessage(<AddDocsDialog />));
+            }}
+          >
+            Add
+          </div>
+        ) : // <div>
+        //   <PlusCircleIcon
+        //     data-tooltip-id={"more-add-docs-button"}
+        //     className="text-vsc-foreground-muted h-5 w-5 cursor-pointer focus:ring-0"
+        //     onClick={() => {
+        //       dispatch(setShowDialog(true));
+        //       dispatch(setDialogMessage(<AddDocsDialog />));
+        //     }}
+        //   />
+        //   <ToolTip id={"more-add-docs-button"} place="top">
+        //     Add Docs
+        //   </ToolTip>
+        // </div>
+        null}
       </div>
       <span className="text-xs text-stone-500">
         Manage your documentation sources

From 263c839153c5c9996f4699389d9b2594badf772d Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Fri, 22 Nov 2024 15:41:41 -0800
Subject: [PATCH 11/17] docs - use config.docs instead of inferring from
 statuses

---
 core/core.ts                                  |   6 +-
 core/index.d.ts                               |   2 -
 core/indexing/docs/DocsService.ts             | 110 ++++++++++--------
 core/protocol/core.ts                         |   3 +-
 core/protocol/passThrough.ts                  |   2 +-
 gui/src/App.tsx                               |   2 +-
 gui/src/components/AccountDialog.tsx          |   2 +
 gui/src/components/dialogs/AddDocsDialog.tsx  |   2 +-
 .../components/indexing/DocsIndexingPeeks.tsx |   6 +-
 ...exingStatus.tsx => DocsIndexingStatus.tsx} |  71 ++++++-----
 .../indexing/DocsIndexingStatuses.tsx         |  57 +++++++++
 .../components/indexing/IndexingStatuses.tsx  |  71 -----------
 gui/src/hooks/useSetup.ts                     |   7 +-
 gui/src/pages/More/More.tsx                   |   4 +-
 gui/src/redux/slices/stateSlice.ts            |   2 +-
 .../{ => nested-folder}/package.json          |   0
 16 files changed, 179 insertions(+), 168 deletions(-)
 rename gui/src/components/indexing/{IndexingStatus.tsx => DocsIndexingStatus.tsx} (68%)
 create mode 100644 gui/src/components/indexing/DocsIndexingStatuses.tsx
 delete mode 100644 gui/src/components/indexing/IndexingStatuses.tsx
 rename manual-testing-sandbox/{ => nested-folder}/package.json (100%)

diff --git a/core/core.ts b/core/core.ts
index 331ffbe857..00ceabb4bd 100644
--- a/core/core.ts
+++ b/core/core.ts
@@ -731,9 +731,6 @@ export class Core {
         this.docsService.setPaused(msg.data.id, msg.data.paused);
       }
     });
-    on("indexing/initStatuses", async (msg) => {
-      return this.docsService.initStatuses();
-    });
     on("docs/getSuggestedDocs", async (msg) => {
       if (hasRequestedDocs) {
         return;
@@ -742,6 +739,9 @@ export class Core {
       const suggestedDocs = await getAllSuggestedDocs(this.ide);
       this.messenger.send("docs/suggestions", suggestedDocs);
     });
+    on("docs/initStatuses", async (msg) => {
+      void this.docsService.initStatuses();
+    });
     //
 
     on("didChangeSelectedProfile", (msg) => {
diff --git a/core/index.d.ts b/core/index.d.ts
index 583e00a855..30aee52901 100644
--- a/core/index.d.ts
+++ b/core/index.d.ts
@@ -68,8 +68,6 @@ export interface IndexingStatus {
   url?: string;
 }
 
-export type IndexingStatusMap = Map<string, IndexingStatus>;
-
 export type PromptTemplateFunction = (
   history: ChatMessage[],
   otherData: Record<string, string>,
diff --git a/core/indexing/docs/DocsService.ts b/core/indexing/docs/DocsService.ts
index 838e8fb542..f9c88cd0ed 100644
--- a/core/indexing/docs/DocsService.ts
+++ b/core/indexing/docs/DocsService.ts
@@ -7,7 +7,6 @@ import {
   ContinueConfig,
   EmbeddingsProvider,
   IDE,
-  IndexingStatusMap,
   IndexingStatus,
   SiteIndexingConfig,
   IdeInfo,
@@ -86,7 +85,6 @@ export type AddParams = {
 export default class DocsService {
   static lanceTableName = "docs";
   static sqlitebTableName = "docs";
-  static indexingType = "docs";
 
   static preIndexedDocsEmbeddingsProvider =
     new TransformersJsEmbeddingsProvider();
@@ -136,38 +134,55 @@ export default class DocsService {
     configHandler.onConfigUpdate(this.handleConfigUpdate.bind(this));
   }
 
-  readonly statuses: IndexingStatusMap = new Map();
-
-  // Function for GUI to retrieve initial pending statuses
-  // And kickoff indexing where needed
-  async initStatuses() {
-    this.config?.docs?.forEach(async (doc) => {
-      const currentStatus = this.statuses.get(doc.startUrl);
-      if (currentStatus) {
-        this.handleStatusUpdate(currentStatus);
-      } else {
-        this.handleStatusUpdate({
-          type: "docs",
-          id: doc.startUrl,
-          embeddingsProviderId: this.config.embeddingsProvider.id,
-          isReindexing: false,
-          progress: 0,
-          description: "Pending",
-          status: "pending",
-          title: doc.title,
-          debugInfo: `max depth: ${doc.maxDepth}`,
-          icon: doc.faviconUrl,
-          url: doc.startUrl,
-        });
-      }
-    });
-  }
+  readonly statuses: Map<string, IndexingStatus> = new Map();
 
   handleStatusUpdate(update: IndexingStatus) {
     this.statuses.set(update.id, update);
     this.messenger?.send("indexing/statusUpdate", update);
   }
 
+  // A way for gui to retrieve initial statuses
+  async initStatuses(): Promise<void> {
+    if (!this.config?.docs) {
+      return;
+    }
+    const metadata = await this.listMetadata();
+
+    this.config.docs?.forEach((doc) => {
+      if (!doc.startUrl) {
+        console.error("Invalid config docs entry", doc);
+        return;
+      }
+
+      const sharedStatus = {
+        type: "docs" as IndexingStatus["type"],
+        id: doc.startUrl,
+        embeddingsProviderId: this.config.embeddingsProvider.id,
+        isReindexing: false,
+        title: doc.title,
+        debugInfo: `max depth: ${doc.maxDepth}`,
+        icon: doc.faviconUrl,
+        url: doc.startUrl,
+      };
+      const indexedStatus: IndexingStatus = metadata.find(
+        (meta) => meta.startUrl === doc.startUrl,
+      )
+        ? {
+            ...sharedStatus,
+            progress: 0,
+            description: "Pending",
+            status: "pending",
+          }
+        : {
+            ...sharedStatus,
+            progress: 1,
+            description: "Complete",
+            status: "complete",
+          };
+      this.handleStatusUpdate(indexedStatus);
+    });
+  }
+
   abort(startUrl: string) {
     const status = this.statuses.get(startUrl);
     if (status) {
@@ -362,6 +377,7 @@ export default class DocsService {
 
     const indexExists = await this.hasMetadata(startUrl);
 
+    // Build status update - most of it is fixed values
     const fixedStatus: Pick<
       IndexingStatus,
       | "type"
@@ -383,6 +399,8 @@ export default class DocsService {
       url: siteIndexingConfig.startUrl,
     };
 
+    // Clear current indexes if reIndexing
+    //
     if (indexExists) {
       if (reIndex) {
         await this.deleteIndexes(startUrl);
@@ -398,6 +416,12 @@ export default class DocsService {
       }
     }
 
+    // If not preindexed
+    const isPreIndexedDoc = !!preIndexedDocs[siteIndexingConfig.startUrl];
+    if (!isPreIndexedDoc) {
+      this.addToConfig(siteIndexingConfig);
+    }
+
     try {
       this.handleStatusUpdate({
         ...fixedStatus,
@@ -435,7 +459,7 @@ export default class DocsService {
         articles.push(article);
 
         processedPages++;
-        await new Promise((resolve) => setTimeout(resolve, 50)); // Locks down GUI if no sleeping
+        await new Promise((resolve) => setTimeout(resolve, 100)); // Locks down GUI if no sleeping
       }
 
       void Telemetry.capture("docs_pages_crawled", {
@@ -788,19 +812,13 @@ export default class DocsService {
         }
       }
 
-      // Sends "Pending" or current status for each
-      await this.initStatuses();
-
-      for (const doc of changedDocs) {
-        console.log(`Updating indexed doc: ${doc.startUrl}`);
-        await this.indexAndAdd(doc, true);
-      }
+      await Promise.allSettled([
+        ...changedDocs.map((doc) => this.indexAndAdd(doc, true)),
+        ...newDocs.map((doc) => this.indexAndAdd(doc)),
+      ]);
 
       for (const doc of newDocs) {
-        console.log(`Indexing new doc: ${doc.startUrl}`);
         void Telemetry.capture("add_docs_config", { url: doc.startUrl });
-
-        await this.indexAndAdd(doc);
       }
 
       for (const doc of deletedDocs) {
@@ -941,7 +959,7 @@ export default class DocsService {
     );
   }
 
-  private addToConfig({ siteIndexingConfig }: AddParams) {
+  private addToConfig(siteIndexingConfig: SiteIndexingConfig) {
     // Handles the case where a user has manually added the doc to config.json
     // so it already exists in the file
     const doesDocExist = this.config.docs?.some(
@@ -959,13 +977,6 @@ export default class DocsService {
   private async add(params: AddParams) {
     await this.addToLance(params);
     await this.addMetadataToSqlite(params);
-
-    const isPreIndexedDoc =
-      !!preIndexedDocs[params.siteIndexingConfig.startUrl];
-
-    if (!isPreIndexedDoc) {
-      this.addToConfig(params);
-    }
   }
 
   // Delete methods
@@ -1030,10 +1041,7 @@ export default class DocsService {
     console.log(
       `Reindexing non-preindexed docs with new embeddings provider: ${embeddingsProvider.id}`,
     );
-    await this.initStatuses();
-    for (const doc of docs) {
-      await this.indexAndAdd(doc, true);
-    }
+    await Promise.allSettled(docs.map((doc) => this.indexAndAdd(doc)));
 
     // Important that this only is invoked after we have successfully
     // cleared and reindex the docs so that the table cannot end up in an
diff --git a/core/protocol/core.ts b/core/protocol/core.ts
index f5b04c9cf5..88cb5642c9 100644
--- a/core/protocol/core.ts
+++ b/core/protocol/core.ts
@@ -9,6 +9,7 @@ import type {
   DiffLine,
   FileSymbolMap,
   IdeSettings,
+  IndexingStatus,
   LLMFullCompletionOptions,
   MessageContent,
   ModelDescription,
@@ -165,8 +166,8 @@ export type ToCoreFromIdeOrWebviewProtocol = {
   "indexing/reindex": [{ type: string; id: string }, void];
   "indexing/abort": [{ type: string; id: string }, void];
   "indexing/setPaused": [{ type: string; id: string; paused: boolean }, void];
-  "indexing/initStatuses": [undefined, void];
   "docs/getSuggestedDocs": [undefined, void];
+  "docs/initStatuses": [undefined, void];
 
   addAutocompleteModel: [{ model: ModelDescription }, void];
 
diff --git a/core/protocol/passThrough.ts b/core/protocol/passThrough.ts
index 2bd8de2c18..720a6f66e8 100644
--- a/core/protocol/passThrough.ts
+++ b/core/protocol/passThrough.ts
@@ -48,11 +48,11 @@ export const WEBVIEW_TO_CORE_PASS_THROUGH: (keyof ToCoreFromWebviewProtocol)[] =
     "index/forceReIndexFiles",
     "index/indexingProgressBarInitialized",
     // Docs, etc.
-    "indexing/initStatuses",
     "indexing/reindex",
     "indexing/abort",
     "indexing/setPaused",
     "docs/getSuggestedDocs",
+    "docs/initStatuses",
     //
     "completeOnboarding",
     "addAutocompleteModel",
diff --git a/gui/src/App.tsx b/gui/src/App.tsx
index 1e34e6e156..ca807a8764 100644
--- a/gui/src/App.tsx
+++ b/gui/src/App.tsx
@@ -8,7 +8,7 @@ import useSubmenuContextProviders from "./hooks/useSubmenuContextProviders";
 import { useVscTheme } from "./hooks/useVscTheme";
 import { AddNewModel, ConfigureProvider } from "./pages/AddNewModel";
 import ConfigErrorPage from "./pages/config-error";
-import Edit from "./pages/Edit";
+import Edit from "./pages/edit";
 import ErrorPage from "./pages/error";
 import Chat from "./pages/gui";
 import History from "./pages/history";
diff --git a/gui/src/components/AccountDialog.tsx b/gui/src/components/AccountDialog.tsx
index db61fe3c0e..5438d11df6 100644
--- a/gui/src/components/AccountDialog.tsx
+++ b/gui/src/components/AccountDialog.tsx
@@ -65,6 +65,8 @@ function AccountDialog() {
     );
   }
 
+  const dispatch = useDispatch();
+
   const changeProfileId = (id: string) => {
     ideMessenger.post("didChangeSelectedProfile", { id });
     dispatch(setSelectedProfileId(id));
diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx
index 68039e85c3..b781a5ec8d 100644
--- a/gui/src/components/dialogs/AddDocsDialog.tsx
+++ b/gui/src/components/dialogs/AddDocsDialog.tsx
@@ -22,7 +22,7 @@ import {
   setShowDialog,
 } from "../../redux/slices/uiStateSlice";
 import { RootState } from "../../redux/store";
-import IndexingStatusViewer from "../indexing/IndexingStatus";
+import IndexingStatusViewer from "../indexing/DocsIndexingStatus";
 
 import { ToolTip } from "../gui/Tooltip";
 import FileIcon from "../FileIcon";
diff --git a/gui/src/components/indexing/DocsIndexingPeeks.tsx b/gui/src/components/indexing/DocsIndexingPeeks.tsx
index e50b4adcf2..d63012865c 100644
--- a/gui/src/components/indexing/DocsIndexingPeeks.tsx
+++ b/gui/src/components/indexing/DocsIndexingPeeks.tsx
@@ -1,4 +1,4 @@
-import { useDispatch } from "react-redux";
+import { useDispatch, useSelector } from "react-redux";
 
 import { IndexingStatus } from "core";
 import { useMemo } from "react";
@@ -8,6 +8,7 @@ import {
   setDialogMessage,
   setShowDialog,
 } from "../../redux/slices/uiStateSlice";
+import { RootState } from "../../redux/store";
 
 export interface DocsIndexingPeekProps {
   status: IndexingStatus;
@@ -58,6 +59,9 @@ interface DocsIndexingPeeksProps {
 }
 
 function DocsIndexingPeekList({ statuses }: DocsIndexingPeeksProps) {
+  const config = useSelector((store: RootState) => store.state.config);
+  const configDocs = config.docs ?? [];
+  // const docs
   return (
     <div className="border-vsc-input-border mt-2 flex flex-col border-0 border-t border-solid pt-2">
       <p className="mx-0 my-1.5 p-0 px-1 font-semibold text-stone-500">
diff --git a/gui/src/components/indexing/IndexingStatus.tsx b/gui/src/components/indexing/DocsIndexingStatus.tsx
similarity index 68%
rename from gui/src/components/indexing/IndexingStatus.tsx
rename to gui/src/components/indexing/DocsIndexingStatus.tsx
index 6fe72c7419..4fb5013cd4 100644
--- a/gui/src/components/indexing/IndexingStatus.tsx
+++ b/gui/src/components/indexing/DocsIndexingStatus.tsx
@@ -1,6 +1,6 @@
-import { IndexingStatus } from "core";
+import { IndexingStatus, SiteIndexingConfig } from "core";
 import { PropsWithChildren, useContext, useMemo } from "react";
-import { useDispatch } from "react-redux";
+import { useDispatch, useSelector } from "react-redux";
 import { usePostHog } from "posthog-js/react";
 import { IdeMessengerContext } from "../../context/IdeMessenger";
 import {
@@ -12,9 +12,10 @@ import {
   XMarkIcon,
 } from "@heroicons/react/24/outline";
 import { updateIndexingStatus } from "../../redux/slices/stateSlice";
+import { RootState } from "../../redux/store";
 
 interface IndexingStatusViewerProps {
-  status: IndexingStatus;
+  docConfig: SiteIndexingConfig;
 }
 
 const STATUS_TO_ICON: Record<IndexingStatus["status"], any> = {
@@ -27,33 +28,43 @@ const STATUS_TO_ICON: Record<IndexingStatus["status"], any> = {
   failed: XMarkIcon, // Since we show an erorr message below
 };
 
-function IndexingStatusViewer({ status }: IndexingStatusViewerProps) {
+function DocsIndexingStatus({ docConfig }: IndexingStatusViewerProps) {
   const ideMessenger = useContext(IdeMessengerContext);
   const posthog = usePostHog();
   const dispatch = useDispatch();
 
+  // const status = undefined;
+  const status = useSelector(
+    (store: RootState) => store.state.indexing.statuses[docConfig.startUrl],
+  );
+
   const reIndex = () =>
     ideMessenger.post("indexing/reindex", {
-      type: status.type,
-      id: status.id,
+      type: "docs",
+      id: docConfig.startUrl,
     });
 
   const abort = () => {
     ideMessenger.post("indexing/abort", {
-      type: status.type,
-      id: status.id,
+      type: "docs",
+      id: docConfig.startUrl,
     });
     // Optimistic abort status
-    dispatch(
-      updateIndexingStatus({ ...status, status: "aborted", progress: 0 }),
-    );
+    if (status) {
+      dispatch(
+        updateIndexingStatus({ ...status, status: "aborted", progress: 0 }),
+      );
+    }
   };
 
   const progressPercentage = useMemo(() => {
+    if (!status) {
+      return 0;
+    }
     return Math.min(100, Math.max(0, status.progress * 100));
-  }, [status.progress]);
+  }, [status?.progress]);
 
-  const Icon = STATUS_TO_ICON[status.status];
+  const Icon = STATUS_TO_ICON[status?.status];
 
   return (
     <div className="mt-2 flex w-full flex-col">
@@ -62,24 +73,26 @@ function IndexingStatusViewer({ status }: IndexingStatusViewerProps) {
         className={`flex flex-row items-center justify-between gap-2 text-sm`}
       >
         <div
-          className={`flex flex-row items-center gap-2 ${status.url ? "cursor-pointer hover:underline" : ""}`}
+          className={`flex flex-row items-center gap-2 ${status?.url ? "cursor-pointer hover:underline" : ""}`}
           onClick={() => {
-            if (status.url) {
+            if (status?.url) {
               ideMessenger.post("openUrl", status.url);
             }
           }}
         >
-          {status.icon ? (
-            <img src={status.icon} alt="doc icon" className="h-4 w-4" />
+          {docConfig.faviconUrl ? (
+            <img
+              src={docConfig.faviconUrl}
+              alt="doc icon"
+              className="h-4 w-4"
+            />
           ) : null}
           <p className="lines lines-1 m-0 p-0 text-left">
-            {status.title ?? status.id}
+            {docConfig.title ?? docConfig.startUrl}
           </p>
-          {!!status.url && (
-            <ArrowTopRightOnSquareIcon className="mb-0.5 h-3 w-3 text-stone-500" />
-          )}
+          <ArrowTopRightOnSquareIcon className="mb-0.5 h-3 w-3 text-stone-500" />
         </div>
-        {status.status === "pending" ? (
+        {status?.status === "pending" ? (
           <div className="text-xs text-stone-500">Pending...</div>
         ) : (
           <div className="flex flex-row items-center gap-1 text-stone-500">
@@ -87,7 +100,7 @@ function IndexingStatusViewer({ status }: IndexingStatusViewerProps) {
             {Icon ? (
               <Icon
                 className={`inline-block h-4 w-4 text-stone-500 ${
-                  status.status === "indexing" ? "animate-spin-slow" : ""
+                  status?.status === "indexing" ? "animate-spin-slow" : ""
                 }`}
               ></Icon>
             ) : null}
@@ -98,7 +111,7 @@ function IndexingStatusViewer({ status }: IndexingStatusViewerProps) {
       <div className="my-2 h-1.5 w-full rounded-md border border-solid border-gray-400">
         <div
           className={`h-full rounded-lg transition-all duration-200 ease-in-out ${
-            status.status === "failed" ? "bg-red-600" : "bg-stone-500"
+            status?.status === "failed" ? "bg-red-600" : "bg-stone-500"
           }`}
           style={{
             width: `${progressPercentage}%`,
@@ -118,7 +131,7 @@ function IndexingStatusViewer({ status }: IndexingStatusViewerProps) {
               paused: () => {},
               deleted: () => {},
               pending: () => {},
-            }[status.status]
+            }[status?.status]
           }
         >
           {
@@ -130,18 +143,16 @@ function IndexingStatusViewer({ status }: IndexingStatusViewerProps) {
               paused: "",
               deleted: "",
               pending: "",
-            }[status.status]
+            }[status?.status]
           }
         </span>
 
-        {/* {status.status === "indexing" && ( */}
         <span className="lines lines-1 text-right text-xs text-stone-500">
-          {status.description}
+          {status?.description}
         </span>
-        {/* )} */}
       </div>
     </div>
   );
 }
 
-export default IndexingStatusViewer;
+export default DocsIndexingStatus;
diff --git a/gui/src/components/indexing/DocsIndexingStatuses.tsx b/gui/src/components/indexing/DocsIndexingStatuses.tsx
new file mode 100644
index 0000000000..84fdba9031
--- /dev/null
+++ b/gui/src/components/indexing/DocsIndexingStatuses.tsx
@@ -0,0 +1,57 @@
+import { useMemo } from "react";
+import { useDispatch, useSelector } from "react-redux";
+import { RootState } from "../../redux/store";
+import IndexingStatusViewer from "./DocsIndexingStatus";
+import { SecondaryButton } from "..";
+import {
+  setDialogMessage,
+  setShowDialog,
+} from "../../redux/slices/uiStateSlice";
+import AddDocsDialog from "../dialogs/AddDocsDialog";
+import DocsIndexingStatus from "./DocsIndexingStatus";
+
+function DocsIndexingStatuses() {
+  const dispatch = useDispatch();
+  const config = useSelector((store: RootState) => store.state.config);
+  const configDocs = config.docs ?? [];
+
+  // const indexingStatuses = useSelector(
+  //   (store: RootState) => store.state.indexing.statuses,
+  // );
+  // const docsStatuses = useMemo(() => {
+  //   const docs = Object.values(indexingStatuses).filter(
+  //     (status) => status.type === "docs" && status.status !== "deleted",
+  //   );
+  //   return docs;
+  // }, [indexingStatuses]);
+
+  return (
+    <div className="flex flex-col gap-1">
+      <div className="flex flex-row items-center justify-between">
+        <h3 className="mb-1 mt-0 text-xl">@docs indexes</h3>
+        {configDocs.length ? (
+          <SecondaryButton
+            className="border-vsc-foreground text-vsc-foreground enabled:hover:bg-vsc-background m-2 rounded border bg-inherit px-3 py-2 enabled:hover:cursor-pointer enabled:hover:opacity-90 disabled:text-gray-500"
+            type="submit"
+            onClick={() => {
+              dispatch(setShowDialog(true));
+              dispatch(setDialogMessage(<AddDocsDialog />));
+            }}
+          >
+            Add
+          </SecondaryButton>
+        ) : null}
+      </div>
+      <span className="text-xs text-stone-500">
+        Manage your documentation sources
+      </span>
+      <div className="flex max-h-[170px] flex-col gap-1 overflow-x-hidden overflow-y-scroll pr-2">
+        {configDocs.map((doc) => {
+          return <DocsIndexingStatus key={doc.startUrl} docConfig={doc} />;
+        })}
+      </div>
+    </div>
+  );
+}
+
+export default DocsIndexingStatuses;
diff --git a/gui/src/components/indexing/IndexingStatuses.tsx b/gui/src/components/indexing/IndexingStatuses.tsx
deleted file mode 100644
index 3ea04216f4..0000000000
--- a/gui/src/components/indexing/IndexingStatuses.tsx
+++ /dev/null
@@ -1,71 +0,0 @@
-import { useMemo } from "react";
-import { useDispatch, useSelector } from "react-redux";
-import { RootState } from "../../redux/store";
-import IndexingStatusViewer from "./IndexingStatus";
-import { Button, SecondaryButton } from "..";
-import {
-  setDialogMessage,
-  setShowDialog,
-} from "../../redux/slices/uiStateSlice";
-import AddDocsDialog from "../dialogs/AddDocsDialog";
-import { PlusCircleIcon } from "@heroicons/react/24/outline";
-import { ToolTip } from "../gui/Tooltip";
-
-function IndexingStatuses() {
-  const indexingStatuses = useSelector(
-    (store: RootState) => store.state.indexing.statuses,
-  );
-  const docsStatuses = useMemo(() => {
-    const docs = Object.values(indexingStatuses).filter(
-      (status) => status.type === "docs" && status.status !== "deleted",
-    );
-    return docs;
-  }, [indexingStatuses]);
-  const dispatch = useDispatch();
-
-  return (
-    <div className="flex flex-col gap-1">
-      <div className="flex flex-row items-center justify-between">
-        <h3 className="mb-1 mt-0 text-xl">@docs indexes</h3>
-        {docsStatuses.length ? (
-          <div
-            className="border-vsc-foreground text-vsc-foreground enabled:hover:bg-vsc-background m-2 rounded border border-solid bg-inherit px-3 py-2 enabled:hover:cursor-pointer enabled:hover:opacity-90 disabled:text-gray-500"
-            onClick={() => {
-              dispatch(setShowDialog(true));
-              dispatch(setDialogMessage(<AddDocsDialog />));
-            }}
-          >
-            Add
-          </div>
-        ) : // <div>
-        //   <PlusCircleIcon
-        //     data-tooltip-id={"more-add-docs-button"}
-        //     className="text-vsc-foreground-muted h-5 w-5 cursor-pointer focus:ring-0"
-        //     onClick={() => {
-        //       dispatch(setShowDialog(true));
-        //       dispatch(setDialogMessage(<AddDocsDialog />));
-        //     }}
-        //   />
-        //   <ToolTip id={"more-add-docs-button"} place="top">
-        //     Add Docs
-        //   </ToolTip>
-        // </div>
-        null}
-      </div>
-      <span className="text-xs text-stone-500">
-        Manage your documentation sources
-      </span>
-      <div className="flex max-h-[170px] flex-col gap-1 overflow-x-hidden overflow-y-scroll pr-2">
-        {docsStatuses.length ? (
-          docsStatuses.map((status) => {
-            return <IndexingStatusViewer key={status.id} status={status} />;
-          })
-        ) : (
-          <SecondaryButton>Add Docs</SecondaryButton>
-        )}
-      </div>
-    </div>
-  );
-}
-
-export default IndexingStatuses;
diff --git a/gui/src/hooks/useSetup.ts b/gui/src/hooks/useSetup.ts
index ecc7a7ac92..34a1e2f748 100644
--- a/gui/src/hooks/useSetup.ts
+++ b/gui/src/hooks/useSetup.ts
@@ -54,9 +54,12 @@ function useSetup(dispatch: Dispatch) {
     loadConfig();
     const interval = setInterval(() => {
       if (initialConfigLoad.current) {
+        // Docs init on config load
+        ideMessenger.post("docs/getSuggestedDocs", undefined);
+        ideMessenger.post("docs/initStatuses", undefined);
+
         // This triggers sending pending status to the GUI for relevant docs indexes
         clearInterval(interval);
-        ideMessenger.post("indexing/initStatuses", undefined);
         return;
       }
       loadConfig();
@@ -85,8 +88,6 @@ function useSetup(dispatch: Dispatch) {
 
   // ON LOAD
   useEffect(() => {
-    ideMessenger.post("docs/getSuggestedDocs", undefined);
-
     // Override persisted state
     dispatch(setInactive());
 
diff --git a/gui/src/pages/More/More.tsx b/gui/src/pages/More/More.tsx
index e743a38343..81648b41b6 100644
--- a/gui/src/pages/More/More.tsx
+++ b/gui/src/pages/More/More.tsx
@@ -15,7 +15,7 @@ import { setOnboardingCard } from "../../redux/slices/uiStateSlice";
 import useHistory from "../../hooks/useHistory";
 import MoreHelpRow from "./MoreHelpRow";
 import IndexingProgress from "./IndexingProgress";
-import IndexingStatuses from "../../components/indexing/IndexingStatuses";
+import DocsIndexingStatuses from "../../components/indexing/DocsIndexingStatuses";
 
 function MorePage() {
   useNavigationListener();
@@ -52,7 +52,7 @@ function MorePage() {
           <IndexingProgress />
         </div>
         <div className="flex flex-col py-5">
-          <IndexingStatuses />
+          <DocsIndexingStatuses />
         </div>
 
         <div className="py-5">
diff --git a/gui/src/redux/slices/stateSlice.ts b/gui/src/redux/slices/stateSlice.ts
index 42fea2a60b..65075d87cd 100644
--- a/gui/src/redux/slices/stateSlice.ts
+++ b/gui/src/redux/slices/stateSlice.ts
@@ -45,7 +45,7 @@ type State = {
   nextCodeBlockToApplyIndex: number;
   indexing: {
     hiddenChatPeekTypes: Record<IndexingStatus["type"], boolean>;
-    statuses: Record<string, IndexingStatus>;
+    statuses: Record<string, IndexingStatus>; // status id -> status
   };
   streamAborter: AbortController;
   docsSuggestions: PackageDocsResult[];
diff --git a/manual-testing-sandbox/package.json b/manual-testing-sandbox/nested-folder/package.json
similarity index 100%
rename from manual-testing-sandbox/package.json
rename to manual-testing-sandbox/nested-folder/package.json

From a541fe2f381a5ca3154a22e9ec885841c0d7ca2e Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Fri, 22 Nov 2024 16:03:44 -0800
Subject: [PATCH 12/17] docs suggestions v2

---
 gui/src/components/dialogs/AddDocsDialog.tsx | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx
index b781a5ec8d..00134a6f73 100644
--- a/gui/src/components/dialogs/AddDocsDialog.tsx
+++ b/gui/src/components/dialogs/AddDocsDialog.tsx
@@ -1,35 +1,25 @@
 import {
-  CheckCircleIcon,
   CheckIcon,
-  ChevronDownIcon,
-  ChevronUpIcon,
-  CodeBracketIcon,
-  ExclamationTriangleIcon,
   InformationCircleIcon,
-  LinkIcon,
   PencilIcon,
-  PlusCircleIcon,
   PlusIcon,
 } from "@heroicons/react/24/outline";
 import { IndexingStatus, PackageDocsResult, SiteIndexingConfig } from "core";
 import { usePostHog } from "posthog-js/react";
 import { useContext, useLayoutEffect, useMemo, useRef, useState } from "react";
 import { useDispatch, useSelector } from "react-redux";
-import { Button, HelperText, Input, lightGray, SecondaryButton } from "..";
+import { Input, SecondaryButton } from "..";
 import { IdeMessengerContext } from "../../context/IdeMessenger";
 import {
   setDialogMessage,
   setShowDialog,
 } from "../../redux/slices/uiStateSlice";
 import { RootState } from "../../redux/store";
-import IndexingStatusViewer from "../indexing/DocsIndexingStatus";
-
 import { ToolTip } from "../gui/Tooltip";
 import FileIcon from "../FileIcon";
 import DocsIndexingPeeks from "../indexing/DocsIndexingPeeks";
 import { updateIndexingStatus } from "../../redux/slices/stateSlice";
 import preIndexedDocs from "core/indexing/docs/preIndexedDocs";
-import { NewSessionButton } from "../mainInput/NewSessionButton";
 
 function AddDocsDialog() {
   const posthog = usePostHog();
@@ -99,6 +89,7 @@ function AddDocsDialog() {
     dispatch(setShowDialog(false));
     dispatch(setDialogMessage(undefined));
   };
+
   function onSubmit(e: any) {
     e.preventDefault();
 

From de984454cd4978cce51ce13a5539a0984b6162b7 Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Mon, 25 Nov 2024 12:25:20 -0800
Subject: [PATCH 13/17] status race condition fix

---
 core/indexing/docs/DocsService.ts | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/core/indexing/docs/DocsService.ts b/core/indexing/docs/DocsService.ts
index f9c88cd0ed..0f3724d61d 100644
--- a/core/indexing/docs/DocsService.ts
+++ b/core/indexing/docs/DocsService.ts
@@ -150,10 +150,15 @@ export default class DocsService {
 
     this.config.docs?.forEach((doc) => {
       if (!doc.startUrl) {
-        console.error("Invalid config docs entry", doc);
+        console.error("Invalid config docs entry, no start");
         return;
       }
 
+      const currentStatus = this.statuses.get(doc.startUrl);
+      if (currentStatus) {
+        return currentStatus;
+      }
+
       const sharedStatus = {
         type: "docs" as IndexingStatus["type"],
         id: doc.startUrl,

From d26f82a3fe11c2aac005f4a8deaa81b2cdc37b74 Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Mon, 25 Nov 2024 16:47:54 -0800
Subject: [PATCH 14/17] edit casing bug

---
 gui/src/App.tsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gui/src/App.tsx b/gui/src/App.tsx
index ca807a8764..82433e2052 100644
--- a/gui/src/App.tsx
+++ b/gui/src/App.tsx
@@ -8,7 +8,7 @@ import useSubmenuContextProviders from "./hooks/useSubmenuContextProviders";
 import { useVscTheme } from "./hooks/useVscTheme";
 import { AddNewModel, ConfigureProvider } from "./pages/AddNewModel";
 import ConfigErrorPage from "./pages/config-error";
-import Edit from "./pages/edit";
+import Edit from "./pages/edit/Edit";
 import ErrorPage from "./pages/error";
 import Chat from "./pages/gui";
 import History from "./pages/history";

From 800e65e51ccc0c978a44bc10de0dedb7b35516dd Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Mon, 25 Nov 2024 17:01:16 -0800
Subject: [PATCH 15/17] tiny tiny

---
 gui/src/App.tsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gui/src/App.tsx b/gui/src/App.tsx
index 82433e2052..ca807a8764 100644
--- a/gui/src/App.tsx
+++ b/gui/src/App.tsx
@@ -8,7 +8,7 @@ import useSubmenuContextProviders from "./hooks/useSubmenuContextProviders";
 import { useVscTheme } from "./hooks/useVscTheme";
 import { AddNewModel, ConfigureProvider } from "./pages/AddNewModel";
 import ConfigErrorPage from "./pages/config-error";
-import Edit from "./pages/edit/Edit";
+import Edit from "./pages/edit";
 import ErrorPage from "./pages/error";
 import Chat from "./pages/gui";
 import History from "./pages/history";

From 7f5ec0dfc10f9151c8292e5f53d29692ebb7b0df Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Mon, 25 Nov 2024 17:36:16 -0800
Subject: [PATCH 16/17] short

---
 gui/src/pages/{Edit => edi}/AddFileButton.tsx         | 0
 gui/src/pages/{Edit => edi}/CodeToEdit.tsx            | 0
 gui/src/pages/{Edit => edi}/CodeToEditListItem.tsx    | 0
 gui/src/pages/{Edit => edi}/Edit.tsx                  | 0
 gui/src/pages/{Edit => edi}/getMultifileEditPrompt.ts | 0
 gui/src/pages/{Edit => edi}/index.tsx                 | 0
 6 files changed, 0 insertions(+), 0 deletions(-)
 rename gui/src/pages/{Edit => edi}/AddFileButton.tsx (100%)
 rename gui/src/pages/{Edit => edi}/CodeToEdit.tsx (100%)
 rename gui/src/pages/{Edit => edi}/CodeToEditListItem.tsx (100%)
 rename gui/src/pages/{Edit => edi}/Edit.tsx (100%)
 rename gui/src/pages/{Edit => edi}/getMultifileEditPrompt.ts (100%)
 rename gui/src/pages/{Edit => edi}/index.tsx (100%)

diff --git a/gui/src/pages/Edit/AddFileButton.tsx b/gui/src/pages/edi/AddFileButton.tsx
similarity index 100%
rename from gui/src/pages/Edit/AddFileButton.tsx
rename to gui/src/pages/edi/AddFileButton.tsx
diff --git a/gui/src/pages/Edit/CodeToEdit.tsx b/gui/src/pages/edi/CodeToEdit.tsx
similarity index 100%
rename from gui/src/pages/Edit/CodeToEdit.tsx
rename to gui/src/pages/edi/CodeToEdit.tsx
diff --git a/gui/src/pages/Edit/CodeToEditListItem.tsx b/gui/src/pages/edi/CodeToEditListItem.tsx
similarity index 100%
rename from gui/src/pages/Edit/CodeToEditListItem.tsx
rename to gui/src/pages/edi/CodeToEditListItem.tsx
diff --git a/gui/src/pages/Edit/Edit.tsx b/gui/src/pages/edi/Edit.tsx
similarity index 100%
rename from gui/src/pages/Edit/Edit.tsx
rename to gui/src/pages/edi/Edit.tsx
diff --git a/gui/src/pages/Edit/getMultifileEditPrompt.ts b/gui/src/pages/edi/getMultifileEditPrompt.ts
similarity index 100%
rename from gui/src/pages/Edit/getMultifileEditPrompt.ts
rename to gui/src/pages/edi/getMultifileEditPrompt.ts
diff --git a/gui/src/pages/Edit/index.tsx b/gui/src/pages/edi/index.tsx
similarity index 100%
rename from gui/src/pages/Edit/index.tsx
rename to gui/src/pages/edi/index.tsx

From f7ce697b3f27d2cf88dc67987c94b1fba75b2fb5 Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Mon, 25 Nov 2024 17:36:32 -0800
Subject: [PATCH 17/17] long

---
 gui/src/pages/{edi => edit}/AddFileButton.tsx         | 0
 gui/src/pages/{edi => edit}/CodeToEdit.tsx            | 0
 gui/src/pages/{edi => edit}/CodeToEditListItem.tsx    | 0
 gui/src/pages/{edi => edit}/Edit.tsx                  | 0
 gui/src/pages/{edi => edit}/getMultifileEditPrompt.ts | 0
 gui/src/pages/{edi => edit}/index.tsx                 | 0
 6 files changed, 0 insertions(+), 0 deletions(-)
 rename gui/src/pages/{edi => edit}/AddFileButton.tsx (100%)
 rename gui/src/pages/{edi => edit}/CodeToEdit.tsx (100%)
 rename gui/src/pages/{edi => edit}/CodeToEditListItem.tsx (100%)
 rename gui/src/pages/{edi => edit}/Edit.tsx (100%)
 rename gui/src/pages/{edi => edit}/getMultifileEditPrompt.ts (100%)
 rename gui/src/pages/{edi => edit}/index.tsx (100%)

diff --git a/gui/src/pages/edi/AddFileButton.tsx b/gui/src/pages/edit/AddFileButton.tsx
similarity index 100%
rename from gui/src/pages/edi/AddFileButton.tsx
rename to gui/src/pages/edit/AddFileButton.tsx
diff --git a/gui/src/pages/edi/CodeToEdit.tsx b/gui/src/pages/edit/CodeToEdit.tsx
similarity index 100%
rename from gui/src/pages/edi/CodeToEdit.tsx
rename to gui/src/pages/edit/CodeToEdit.tsx
diff --git a/gui/src/pages/edi/CodeToEditListItem.tsx b/gui/src/pages/edit/CodeToEditListItem.tsx
similarity index 100%
rename from gui/src/pages/edi/CodeToEditListItem.tsx
rename to gui/src/pages/edit/CodeToEditListItem.tsx
diff --git a/gui/src/pages/edi/Edit.tsx b/gui/src/pages/edit/Edit.tsx
similarity index 100%
rename from gui/src/pages/edi/Edit.tsx
rename to gui/src/pages/edit/Edit.tsx
diff --git a/gui/src/pages/edi/getMultifileEditPrompt.ts b/gui/src/pages/edit/getMultifileEditPrompt.ts
similarity index 100%
rename from gui/src/pages/edi/getMultifileEditPrompt.ts
rename to gui/src/pages/edit/getMultifileEditPrompt.ts
diff --git a/gui/src/pages/edi/index.tsx b/gui/src/pages/edit/index.tsx
similarity index 100%
rename from gui/src/pages/edi/index.tsx
rename to gui/src/pages/edit/index.tsx