From cb071a74194f656a462f53ed503ba725c31958e3 Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Thu, 21 Nov 2024 00:42:09 -0800
Subject: [PATCH 01/17] package file parsers
---
core/core.ts | 4 +
core/indexing/docs/suggestions/index.ts | 100 +++++++++++++++++
.../suggestions/packageCrawlers/Python.ts | 33 ++++++
.../docs/suggestions/packageCrawlers/TsJs.ts | 33 ++++++
core/protocol/core.ts | 1 +
core/protocol/passThrough.ts | 1 +
extensions/vscode/package-lock.json | 4 +-
gui/src/hooks/useSetup.ts | 3 +-
manual-testing-sandbox/package.json | 102 ++++++++++++++++++
manual-testing-sandbox/requirements.txt | 10 ++
10 files changed, 287 insertions(+), 4 deletions(-)
create mode 100644 core/indexing/docs/suggestions/index.ts
create mode 100644 core/indexing/docs/suggestions/packageCrawlers/Python.ts
create mode 100644 core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
create mode 100644 manual-testing-sandbox/package.json
create mode 100644 manual-testing-sandbox/requirements.txt
diff --git a/core/core.ts b/core/core.ts
index c6b336c7d0..d080ec34b0 100644
--- a/core/core.ts
+++ b/core/core.ts
@@ -40,6 +40,7 @@ import { TTS } from "./util/tts";
import type { ContextItemId, IDE, IndexingProgressUpdate } from ".";
import type { FromCoreProtocol, ToCoreProtocol } from "./protocol";
import type { IMessenger, Message } from "./util/messenger";
+import { getAllSuggestedDocs } from "./indexing/docs/suggestions";
export class Core {
// implements IMessenger<ToCoreProtocol, FromCoreProtocol>
@@ -737,6 +738,9 @@ export class Core {
on("indexing/initStatuses", async (msg) => {
return this.docsService.initStatuses();
});
+ on("docs/getSuggestedDocs", async (msg) => {
+ getAllSuggestedDocs(this.ide);
+ });
//
on("didChangeSelectedProfile", (msg) => {
diff --git a/core/indexing/docs/suggestions/index.ts b/core/indexing/docs/suggestions/index.ts
new file mode 100644
index 0000000000..d3d68aca29
--- /dev/null
+++ b/core/indexing/docs/suggestions/index.ts
@@ -0,0 +1,100 @@
+// write me an interface PackageCrawler that contains:
+// 1. property `language` to store a given language like "python" or "typescript"
+// 2. has a method `getPackageFiles` which takes a list of file names and decides which ones match package/dependency files (e.g. package.json for typescript, requirements.txt for python, etc)
+// 3. has a method `parsePackageFile` which returns a list of package name and version from a relevant package file, in a standardized format like semver
+// 4. has a method `getDocumentationLink` to check for documentation link for a given package (e.g. GET `https://registry.npmjs.org/<package>` and find docs field for typescript, documentation link in the package metadata for PyPi, etc.)
+// Then, write typescript classes to implement this typescript interface for the languages "python" and "typescript"
+
+import { IDE } from "../../..";
+import { walkDir } from "../../walkDir";
+import { PythonPackageCrawler } from "./packageCrawlers/Python";
+import { TypeScriptPackageCrawler } from "./packageCrawlers/TsJs";
+
+const PACKAGE_CRAWLERS = [TypeScriptPackageCrawler, PythonPackageCrawler];
+
+export interface PackageCrawler {
+ language: string;
+ getPackageFiles(fileNames: string[]): string[];
+ parsePackageFile(fileContent: string, filePath: string): PackageInfo[];
+ getDocumentationLink(packageName: string): Promise<PackageDocsResult>;
+}
+
+export type PackageInfo = {
+ name: string;
+ version: string;
+ foundInFilepath: string;
+};
+export type PackageDocsResult = PackageInfo &
+ ({ error: string; link?: never } | { link: string; error?: never });
+
+export async function getAllSuggestedDocs(ide: IDE) {
+ const workspaceDirs = await ide.getWorkspaceDirs();
+ const results = await Promise.all(
+ workspaceDirs.map((dir) => {
+ return walkDir(dir, ide);
+ }),
+ );
+ const allPaths = results.flat(); // TODO only get files, not dirs. Not critical for now
+ const allFiles = allPaths.map((path) => path.split("/").pop()!);
+ const packageFilesByLanguage: Record<string, string[]> = {};
+ for (const Crawler of PACKAGE_CRAWLERS) {
+ const crawler = new Crawler();
+ const packageFilePaths = crawler.getPackageFiles(allFiles);
+ packageFilesByLanguage[crawler.language] = packageFilePaths;
+ }
+
+ const uniqueFilePaths = Array.from(
+ new Set(Object.values(packageFilesByLanguage).flat()),
+ );
+ const fileContentsArray = await Promise.all(
+ uniqueFilePaths.map(async (path) => {
+ const contents = await ide.readFile(path);
+ return { path, contents };
+ }),
+ );
+ const fileContents = new Map(
+ fileContentsArray.map(({ path, contents }) => [path, contents]),
+ );
+
+ const packagesByLanguage: Record<string, PackageInfo[]> = {};
+ PACKAGE_CRAWLERS.forEach((Crawler) => {
+ const crawler = new Crawler();
+ const packageFiles = packageFilesByLanguage[crawler.language];
+ packageFiles.forEach((file) => {
+ const contents = fileContents.get(file);
+ if (!contents) {
+ return;
+ }
+ const packages = crawler.parsePackageFile(contents, file);
+ if (!packagesByLanguage[crawler.language]) {
+ packagesByLanguage[crawler.language] = [];
+ }
+ packagesByLanguage[crawler.language].push(...packages);
+ });
+ });
+ console.log(packagesByLanguage);
+ // const allPackages = await Promise.all(
+ // PACKAGE_CRAWLERS.map(async (Crawler) => {
+ // const crawler = new Crawler();
+ // const packageInfos = uniqueFileContents
+ // .filter(({ path }) => languageToFilePaths[crawler.language].includes(path))
+ // .map(({ path, contents }) => crawler.parsePackageFile(contents, path))
+ // .flat();
+ // return packageInfos;
+ // })
+ // );
+ // for (const Crawler of PACKAGE_CRAWLERS) {
+ // const crawler = new Crawler();
+ // const packages = crawler.parsePackageFile()
+ // languageToFilePaths[crawler.language] = packageFilePaths;
+ // }
+}
+
+// I want to present the user with a list of dependencies and allow them to select which ones to index (embed) documentation for.
+// In order to prevent duplicate file reads, the process will be like this:
+// 1. take in a list of filepaths called `filepaths`
+// 2. loop an array of PackageCrawler classes to build a map of `language` (string) to `packageFilePaths` (string[])
+// 3. Get unique filepaths from `packageFilePaths` and build a map ` of filepath to file contents using an existing `readFile` function, and skipping file reads of already in the map
+// Finally,
+// Add a `` method to the interface and classes that returns
+// Then, assemble the classes in an array, and write a function getAllSuggestedDocs that returns a map of `language` to an ar
diff --git a/core/indexing/docs/suggestions/packageCrawlers/Python.ts b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
new file mode 100644
index 0000000000..ac06997663
--- /dev/null
+++ b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
@@ -0,0 +1,33 @@
+import { PackageCrawler, PackageDocsResult, PackageInfo } from "..";
+
+export class PythonPackageCrawler implements PackageCrawler {
+ language = "python";
+
+ getPackageFiles(fileNames: string[]): string[] {
+ // For Python, we typically look for files like requirements.txt or Pipfile
+ return fileNames.filter(
+ (fileName) => fileName === "requirements.txt" || fileName === "Pipfile",
+ );
+ }
+
+ parsePackageFile(fileContent: string, filepath: string): PackageInfo[] {
+ // Assume the fileContent is a string from a requirements.txt formatted file
+ return fileContent
+ .split("\n")
+ .map((line) => {
+ const [name, version] = line.split("==");
+ return { name, version, foundInFilepath: filepath };
+ })
+ .filter((pkg) => pkg.name && pkg.version);
+ }
+
+ async getDocumentationLink(packageName: string): Promise<PackageDocsResult> {
+ // Fetch metadata from PyPI to find the documentation link
+ const response = await fetch(`https://pypi.org/pypi/${packageName}/json`);
+ if (!response.ok) {
+ throw new Error(`Could not fetch data for package ${packageName}`);
+ }
+ const data = await response.json();
+ return data.info.project_urls?.Documentation;
+ }
+}
diff --git a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
new file mode 100644
index 0000000000..fb073e2a1d
--- /dev/null
+++ b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
@@ -0,0 +1,33 @@
+import { PackageCrawler, PackageDocsResult, PackageInfo } from "..";
+
+export class TypeScriptPackageCrawler implements PackageCrawler {
+ language = "typescript";
+
+ getPackageFiles(fileNames: string[]): string[] {
+ // For TypeScript, we look for package.json file
+ return fileNames.filter((fileName) => fileName === "package.json");
+ }
+
+ parsePackageFile(fileContent: string, filepath: string): PackageInfo[] {
+ // Parse the package.json content
+ const jsonData = JSON.parse(fileContent) as Record<string, Object>;
+ const dependencies = Object.entries(jsonData.dependencies || {}).concat(
+ Object.entries(jsonData.devDependencies || {}),
+ );
+ return dependencies.map(([name, version]) => ({
+ name,
+ version,
+ foundInFilepath: filepath,
+ }));
+ }
+
+ async getDocumentationLink(packageName: string): Promise<PackageDocsResult> {
+ // Fetch metadata from the NPM registry to find the documentation link
+ const response = await fetch(`https://registry.npmjs.org/${packageName}`);
+ if (!response.ok) {
+ throw new Error(`Could not fetch data for package ${packageName}`);
+ }
+ const data = await response.json();
+ return data.homepage;
+ }
+}
diff --git a/core/protocol/core.ts b/core/protocol/core.ts
index c757e5d347..069f9e297a 100644
--- a/core/protocol/core.ts
+++ b/core/protocol/core.ts
@@ -167,6 +167,7 @@ export type ToCoreFromIdeOrWebviewProtocol = {
"indexing/abort": [{ type: string; id: string }, void];
"indexing/setPaused": [{ type: string; id: string; paused: boolean }, void];
"indexing/initStatuses": [undefined, void];
+ "docs/getSuggestedDocs": [undefined, void];
addAutocompleteModel: [{ model: ModelDescription }, void];
diff --git a/core/protocol/passThrough.ts b/core/protocol/passThrough.ts
index 40d537d41b..ebe2a86977 100644
--- a/core/protocol/passThrough.ts
+++ b/core/protocol/passThrough.ts
@@ -52,6 +52,7 @@ export const WEBVIEW_TO_CORE_PASS_THROUGH: (keyof ToCoreFromWebviewProtocol)[] =
"indexing/reindex",
"indexing/abort",
"indexing/setPaused",
+ "docs/getSuggestedDocs",
//
"completeOnboarding",
"addAutocompleteModel",
diff --git a/extensions/vscode/package-lock.json b/extensions/vscode/package-lock.json
index 9405035051..b30a951dfc 100644
--- a/extensions/vscode/package-lock.json
+++ b/extensions/vscode/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "continue",
- "version": "0.9.232",
+ "version": "0.9.233",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "continue",
- "version": "0.9.232",
+ "version": "0.9.233",
"license": "Apache-2.0",
"dependencies": {
"@electron/rebuild": "^3.2.10",
diff --git a/gui/src/hooks/useSetup.ts b/gui/src/hooks/useSetup.ts
index 4bef9da9ce..f183798890 100644
--- a/gui/src/hooks/useSetup.ts
+++ b/gui/src/hooks/useSetup.ts
@@ -84,6 +84,7 @@ function useSetup(dispatch: Dispatch) {
// ON LOAD
useEffect(() => {
+ ideMessenger.post("docs/getSuggestedDocs", undefined);
// Override persisted state
dispatch(setInactive());
@@ -178,8 +179,6 @@ function useSetup(dispatch: Dispatch) {
},
[defaultModelTitle],
);
-
-
}
export default useSetup;
diff --git a/manual-testing-sandbox/package.json b/manual-testing-sandbox/package.json
new file mode 100644
index 0000000000..ef77ac910c
--- /dev/null
+++ b/manual-testing-sandbox/package.json
@@ -0,0 +1,102 @@
+{
+ "name": "gui",
+ "private": true,
+ "type": "module",
+ "author": "Continue Dev, Inc",
+ "license": "Apache-2.0",
+ "scripts": {
+ "dev": "vite",
+ "tsc:check": "tsc -p ./ --noEmit",
+ "build": "tsc && vite build",
+ "preview": "vite preview",
+ "test": "vitest run",
+ "test:coverage": "vitest run --coverage",
+ "test:ui": "vitest --ui",
+ "test:watch": "vitest"
+ },
+ "dependencies": {
+ "@headlessui/react": "^1.7.17",
+ "@heroicons/react": "^2.0.18",
+ "@monaco-editor/react": "^4.6.0",
+ "@reduxjs/toolkit": "^1.9.3",
+ "@tiptap/core": "^2.3.2",
+ "@tiptap/extension-document": "^2.3.2",
+ "@tiptap/extension-dropcursor": "^2.1.16",
+ "@tiptap/extension-history": "^2.3.2",
+ "@tiptap/extension-image": "^2.1.16",
+ "@tiptap/extension-mention": "^2.1.13",
+ "@tiptap/extension-paragraph": "^2.3.2",
+ "@tiptap/extension-placeholder": "^2.1.13",
+ "@tiptap/extension-text": "^2.3.2",
+ "@tiptap/pm": "^2.1.13",
+ "@tiptap/react": "^2.1.13",
+ "@tiptap/starter-kit": "^2.1.13",
+ "@tiptap/suggestion": "^2.1.13",
+ "@types/vscode-webview": "^1.57.1",
+ "core": "file:../core",
+ "dompurify": "^3.0.6",
+ "downshift": "^7.6.0",
+ "lodash": "^4.17.21",
+ "minisearch": "^7.0.2",
+ "onigasm": "^2.2.5",
+ "posthog-js": "^1.130.1",
+ "prismjs": "^1.29.0",
+ "react": "^18.2.0",
+ "react-dom": "^18.2.0",
+ "react-error-boundary": "^4.0.11",
+ "react-hook-form": "^7.47.0",
+ "react-intersection-observer": "^9.13.1",
+ "react-markdown": "^9.0.1",
+ "react-redux": "^8.0.5",
+ "react-remark": "^2.1.0",
+ "react-router-dom": "^6.14.2",
+ "react-switch": "^7.0.0",
+ "react-syntax-highlighter": "^15.5.0",
+ "react-tooltip": "^5.18.0",
+ "redux-persist": "^6.0.0",
+ "redux-persist-transform-filter": "^0.0.22",
+ "rehype-highlight": "^7.0.0",
+ "rehype-katex": "^7.0.0",
+ "rehype-wrap-all": "^1.1.0",
+ "remark-math": "^6.0.0",
+ "reselect": "^5.1.1",
+ "seti-file-icons": "^0.0.8",
+ "socket.io-client": "^4.7.2",
+ "styled-components": "^5.3.6",
+ "table": "^6.8.1",
+ "tippy.js": "^6.3.7",
+ "unist-util-visit": "^5.0.0",
+ "uuid": "^9.0.1",
+ "vscode-webview": "^1.0.1-beta.1"
+ },
+ "devDependencies": {
+ "@swc/cli": "^0.3.14",
+ "@swc/core": "^1.7.26",
+ "@testing-library/dom": "^10.4.0",
+ "@testing-library/jest-dom": "^6.5.0",
+ "@testing-library/react": "^16.0.1",
+ "@testing-library/user-event": "^14.5.2",
+ "@types/lodash": "^4.17.6",
+ "@types/node": "^20.5.6",
+ "@types/node-fetch": "^2.6.4",
+ "@types/react": "^18.3.11",
+ "@types/react-dom": "^18.3.1",
+ "@types/react-router-dom": "^5.3.3",
+ "@types/react-syntax-highlighter": "^15.5.7",
+ "@types/styled-components": "^5.1.26",
+ "@vitejs/plugin-react-swc": "^3.7.0",
+ "@vitest/coverage-v8": "^2.1.3",
+ "@vitest/ui": "^2.1.3",
+ "autoprefixer": "^10.4.13",
+ "jsdom": "^25.0.1",
+ "postcss": "^8.4.21",
+ "tailwindcss": "^3.2.7",
+ "typescript": "^4.9.3",
+ "vite": "^4.1.0",
+ "vitest": "^2.1.3"
+ },
+ "engine-strict": true,
+ "engines": {
+ "node": ">=20.11.0"
+ }
+}
diff --git a/manual-testing-sandbox/requirements.txt b/manual-testing-sandbox/requirements.txt
new file mode 100644
index 0000000000..793d6a8275
--- /dev/null
+++ b/manual-testing-sandbox/requirements.txt
@@ -0,0 +1,10 @@
+flask==2.1.1
+requests==2.28.1
+numpy==1.23.4
+pandas==1.5.0
+scipy==1.9.3
+django==4.1.3
+matplotlib==3.6.2
+pytest==7.2.0
+sqlalchemy==1.4.41
+beautifulsoup4==4.11.1
\ No newline at end of file
From 954bd5f563a278c04f40f0ec40bfbe27fc497afb Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Thu, 21 Nov 2024 11:58:48 -0800
Subject: [PATCH 02/17] python and typescript crawlers
---
core/index.d.ts | 29 +++++
core/indexing/docs/suggestions/index.ts | 115 +++++++++++-------
.../suggestions/packageCrawlers/Python.ts | 39 ++++--
.../docs/suggestions/packageCrawlers/TsJs.ts | 40 ++++--
4 files changed, 159 insertions(+), 64 deletions(-)
diff --git a/core/index.d.ts b/core/index.d.ts
index 0646edc541..de1adb8934 100644
--- a/core/index.d.ts
+++ b/core/index.d.ts
@@ -1190,3 +1190,32 @@ export interface BrowserSerializedContinueConfig {
experimental?: ExperimentalConfig;
analytics?: AnalyticsConfig;
}
+
+// DOCS SUGGESTIONS AND PACKAGE INFO
+export interface FilePathAndName {
+ path: string;
+ name: string;
+}
+export type ParsedPackageInfo = {
+ language: string;
+ name: string;
+ version: string;
+ packageFile: FilePathAndName;
+};
+
+export type PackageDetails = {
+ docsLink?: string;
+ title?: string;
+ description?: string;
+};
+
+export type PackageDetailsSuccess = PackageDetails & {
+ docsLink: string;
+};
+
+export type PackageDocsResult = {
+ packageInfo: ParsedPackageInfo;
+} & (
+ | { error: string; details?: never }
+ | { details: PackageDetailsSuccess; error?: never }
+);
diff --git a/core/indexing/docs/suggestions/index.ts b/core/indexing/docs/suggestions/index.ts
index d3d68aca29..5699cd761c 100644
--- a/core/indexing/docs/suggestions/index.ts
+++ b/core/indexing/docs/suggestions/index.ts
@@ -1,12 +1,12 @@
-// write me an interface PackageCrawler that contains:
-// 1. property `language` to store a given language like "python" or "typescript"
-// 2. has a method `getPackageFiles` which takes a list of file names and decides which ones match package/dependency files (e.g. package.json for typescript, requirements.txt for python, etc)
-// 3. has a method `parsePackageFile` which returns a list of package name and version from a relevant package file, in a standardized format like semver
-// 4. has a method `getDocumentationLink` to check for documentation link for a given package (e.g. GET `https://registry.npmjs.org/<package>` and find docs field for typescript, documentation link in the package metadata for PyPi, etc.)
-// Then, write typescript classes to implement this typescript interface for the languages "python" and "typescript"
-
-import { IDE } from "../../..";
+import {
+ FilePathAndName,
+ IDE,
+ PackageDetails,
+ PackageDocsResult,
+ ParsedPackageInfo,
+} from "../../..";
import { walkDir } from "../../walkDir";
+
import { PythonPackageCrawler } from "./packageCrawlers/Python";
import { TypeScriptPackageCrawler } from "./packageCrawlers/TsJs";
@@ -14,19 +14,14 @@ const PACKAGE_CRAWLERS = [TypeScriptPackageCrawler, PythonPackageCrawler];
export interface PackageCrawler {
language: string;
- getPackageFiles(fileNames: string[]): string[];
- parsePackageFile(fileContent: string, filePath: string): PackageInfo[];
- getDocumentationLink(packageName: string): Promise<PackageDocsResult>;
+ getPackageFiles(files: FilePathAndName[]): FilePathAndName[];
+ parsePackageFile(
+ file: FilePathAndName,
+ contents: string,
+ ): ParsedPackageInfo[];
+ getPackageDetails(packageInfo: ParsedPackageInfo): Promise<PackageDetails>;
}
-export type PackageInfo = {
- name: string;
- version: string;
- foundInFilepath: string;
-};
-export type PackageDocsResult = PackageInfo &
- ({ error: string; link?: never } | { link: string; error?: never });
-
export async function getAllSuggestedDocs(ide: IDE) {
const workspaceDirs = await ide.getWorkspaceDirs();
const results = await Promise.all(
@@ -35,16 +30,26 @@ export async function getAllSuggestedDocs(ide: IDE) {
}),
);
const allPaths = results.flat(); // TODO only get files, not dirs. Not critical for now
- const allFiles = allPaths.map((path) => path.split("/").pop()!);
- const packageFilesByLanguage: Record<string, string[]> = {};
+ const allFiles = allPaths.map((path) => ({
+ path,
+ name: path.split(/[\\/]/).pop()!,
+ }));
+
+ // Build map of language -> package files
+ const packageFilesByLanguage: Record<string, FilePathAndName[]> = {};
for (const Crawler of PACKAGE_CRAWLERS) {
const crawler = new Crawler();
const packageFilePaths = crawler.getPackageFiles(allFiles);
packageFilesByLanguage[crawler.language] = packageFilePaths;
}
+ // Get file contents for all unique package files
const uniqueFilePaths = Array.from(
- new Set(Object.values(packageFilesByLanguage).flat()),
+ new Set(
+ Object.values(packageFilesByLanguage).flatMap((files) =>
+ files.map((file) => file.path),
+ ),
+ ),
);
const fileContentsArray = await Promise.all(
uniqueFilePaths.map(async (path) => {
@@ -56,40 +61,68 @@ export async function getAllSuggestedDocs(ide: IDE) {
fileContentsArray.map(({ path, contents }) => [path, contents]),
);
- const packagesByLanguage: Record<string, PackageInfo[]> = {};
+ // Parse package files and build map of language -> packages
+ const packagesByLanguage: Record<string, ParsedPackageInfo[]> = {};
PACKAGE_CRAWLERS.forEach((Crawler) => {
const crawler = new Crawler();
const packageFiles = packageFilesByLanguage[crawler.language];
packageFiles.forEach((file) => {
- const contents = fileContents.get(file);
+ const contents = fileContents.get(file.path);
if (!contents) {
return;
}
- const packages = crawler.parsePackageFile(contents, file);
+ const packages = crawler.parsePackageFile(file, contents);
if (!packagesByLanguage[crawler.language]) {
packagesByLanguage[crawler.language] = [];
}
packagesByLanguage[crawler.language].push(...packages);
});
});
- console.log(packagesByLanguage);
- // const allPackages = await Promise.all(
- // PACKAGE_CRAWLERS.map(async (Crawler) => {
- // const crawler = new Crawler();
- // const packageInfos = uniqueFileContents
- // .filter(({ path }) => languageToFilePaths[crawler.language].includes(path))
- // .map(({ path, contents }) => crawler.parsePackageFile(contents, path))
- // .flat();
- // return packageInfos;
- // })
- // );
- // for (const Crawler of PACKAGE_CRAWLERS) {
- // const crawler = new Crawler();
- // const packages = crawler.parsePackageFile()
- // languageToFilePaths[crawler.language] = packageFilePaths;
- // }
+
+ // Get documentation links for all packages
+ const docsByLanguage: Record<string, PackageDocsResult[]> = {};
+ await Promise.all(
+ PACKAGE_CRAWLERS.map(async (Crawler) => {
+ const crawler = new Crawler();
+ const packages = packagesByLanguage[crawler.language];
+ docsByLanguage[crawler.language] = await Promise.all(
+ packages.map(async (packageInfo) => {
+ try {
+ const details = await crawler.getPackageDetails(packageInfo);
+ if (!details.docsLink) {
+ return {
+ packageInfo,
+ error: `No documentation link found for ${packageInfo.name}`,
+ };
+ }
+ return {
+ packageInfo,
+ details: {
+ ...details,
+ docsLink: details.docsLink,
+ },
+ };
+ } catch (error) {
+ return {
+ packageInfo,
+ error: `Error getting package details for ${name}`,
+ };
+ }
+ }),
+ );
+ }),
+ );
+
+ return docsByLanguage;
}
+// write me an interface PackageCrawler that contains:
+// 1. property `language` to store a given language like "python" or "typescript"
+// 2. has a method `getPackageFiles` which takes a list of file names and decides which ones match package/dependency files (e.g. package.json for typescript, requirements.txt for python, etc)
+// 3. has a method `parsePackageFile` which returns a list of package name and version from a relevant package file, in a standardized format like semver
+// 4. has a method `getDocumentationLink` to check for documentation link for a given package (e.g. GET `https://registry.npmjs.org/<package>` and find docs field for typescript, documentation link in the package metadata for PyPi, etc.)
+// Then, write typescript classes to implement this typescript interface for the languages "python" and "typescript"
+
// I want to present the user with a list of dependencies and allow them to select which ones to index (embed) documentation for.
// In order to prevent duplicate file reads, the process will be like this:
// 1. take in a list of filepaths called `filepaths`
diff --git a/core/indexing/docs/suggestions/packageCrawlers/Python.ts b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
index ac06997663..233e444285 100644
--- a/core/indexing/docs/suggestions/packageCrawlers/Python.ts
+++ b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
@@ -1,33 +1,50 @@
-import { PackageCrawler, PackageDocsResult, PackageInfo } from "..";
+import { PackageCrawler } from "..";
+import {
+ FilePathAndName,
+ PackageDetails,
+ ParsedPackageInfo,
+} from "../../../..";
export class PythonPackageCrawler implements PackageCrawler {
language = "python";
- getPackageFiles(fileNames: string[]): string[] {
+ getPackageFiles(files: FilePathAndName[]): FilePathAndName[] {
// For Python, we typically look for files like requirements.txt or Pipfile
- return fileNames.filter(
- (fileName) => fileName === "requirements.txt" || fileName === "Pipfile",
+ return files.filter(
+ (file) => file.name === "requirements.txt" || file.name === "Pipfile",
);
}
- parsePackageFile(fileContent: string, filepath: string): PackageInfo[] {
+ parsePackageFile(
+ file: FilePathAndName,
+ contents: string,
+ ): ParsedPackageInfo[] {
// Assume the fileContent is a string from a requirements.txt formatted file
- return fileContent
+ return contents
.split("\n")
.map((line) => {
const [name, version] = line.split("==");
- return { name, version, foundInFilepath: filepath };
+ return { name, version, packageFile: file, language: this.language };
})
.filter((pkg) => pkg.name && pkg.version);
}
- async getDocumentationLink(packageName: string): Promise<PackageDocsResult> {
+ async getPackageDetails(
+ packageInfo: ParsedPackageInfo,
+ ): Promise<PackageDetails> {
// Fetch metadata from PyPI to find the documentation link
- const response = await fetch(`https://pypi.org/pypi/${packageName}/json`);
+
+ const response = await fetch(
+ `https://pypi.org/pypi/${packageInfo.name}/json`,
+ );
if (!response.ok) {
- throw new Error(`Could not fetch data for package ${packageName}`);
+ throw new Error(`Could not fetch data for package ${packageInfo.name}`);
}
const data = await response.json();
- return data.info.project_urls?.Documentation;
+ return {
+ docsLink: data.info.project_urls?.Documentation as string | undefined,
+ // title: data.info.name,
+ // description: data.info.summary as string | undefined,
+ };
}
}
diff --git a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
index fb073e2a1d..107d9ac2a1 100644
--- a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
+++ b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
@@ -1,33 +1,49 @@
-import { PackageCrawler, PackageDocsResult, PackageInfo } from "..";
+import { PackageCrawler } from "..";
+import {
+ FilePathAndName,
+ PackageDetails,
+ ParsedPackageInfo,
+} from "../../../..";
export class TypeScriptPackageCrawler implements PackageCrawler {
- language = "typescript";
+ language = "js-ts";
- getPackageFiles(fileNames: string[]): string[] {
- // For TypeScript, we look for package.json file
- return fileNames.filter((fileName) => fileName === "package.json");
+ getPackageFiles(files: FilePathAndName[]): FilePathAndName[] {
+ // For Javascript/TypeScript, we look for package.json file
+ return files.filter((file) => file.name === "package.json");
}
- parsePackageFile(fileContent: string, filepath: string): PackageInfo[] {
+ parsePackageFile(
+ file: FilePathAndName,
+ contents: string,
+ ): ParsedPackageInfo[] {
// Parse the package.json content
- const jsonData = JSON.parse(fileContent) as Record<string, Object>;
+ const jsonData = JSON.parse(contents) as Record<string, Object>;
const dependencies = Object.entries(jsonData.dependencies || {}).concat(
Object.entries(jsonData.devDependencies || {}),
);
return dependencies.map(([name, version]) => ({
name,
version,
- foundInFilepath: filepath,
+ packageFile: file,
+ language: this.language,
}));
}
- async getDocumentationLink(packageName: string): Promise<PackageDocsResult> {
+ async getPackageDetails(
+ packageInfo: ParsedPackageInfo,
+ ): Promise<PackageDetails> {
+ const { name } = packageInfo;
// Fetch metadata from the NPM registry to find the documentation link
- const response = await fetch(`https://registry.npmjs.org/${packageName}`);
+ const response = await fetch(`https://registry.npmjs.org/${name}`);
if (!response.ok) {
- throw new Error(`Could not fetch data for package ${packageName}`);
+ throw new Error(`Could not fetch data for package ${name}`);
}
const data = await response.json();
- return data.homepage;
+ return {
+ docsLink: data.homepage as string | undefined,
+ // title: data.name,
+ // description: data.description as string | undefined,
+ };
}
}
From abd46dfb4ff1aea84e94922fdda5d6fce23c6edf Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Thu, 21 Nov 2024 12:52:25 -0800
Subject: [PATCH 03/17] docs suggestions: packages in GUI
---
core/core.ts | 3 +-
core/index.d.ts | 3 ++
core/indexing/docs/suggestions/index.ts | 4 +--
core/protocol/passThrough.ts | 1 +
core/protocol/webview.ts | 2 ++
gui/src/components/dialogs/AddDocsDialog.tsx | 37 ++++++++++++++++++--
gui/src/hooks/useSetup.ts | 6 ++++
gui/src/redux/slices/stateSlice.ts | 11 ++++++
8 files changed, 61 insertions(+), 6 deletions(-)
diff --git a/core/core.ts b/core/core.ts
index d080ec34b0..d81df8a379 100644
--- a/core/core.ts
+++ b/core/core.ts
@@ -739,7 +739,8 @@ export class Core {
return this.docsService.initStatuses();
});
on("docs/getSuggestedDocs", async (msg) => {
- getAllSuggestedDocs(this.ide);
+ const suggestedDocs = await getAllSuggestedDocs(this.ide);
+ this.messenger.send("docs/suggestions", suggestedDocs);
});
//
diff --git a/core/index.d.ts b/core/index.d.ts
index de1adb8934..d6b2c1970c 100644
--- a/core/index.d.ts
+++ b/core/index.d.ts
@@ -1219,3 +1219,6 @@ export type PackageDocsResult = {
| { error: string; details?: never }
| { details: PackageDetailsSuccess; error?: never }
);
+
+// language -> package -> package info
+export type DocsSuggestions = Record<string, PackageDocsResult[]>;
diff --git a/core/indexing/docs/suggestions/index.ts b/core/indexing/docs/suggestions/index.ts
index 5699cd761c..3ee4fb6426 100644
--- a/core/indexing/docs/suggestions/index.ts
+++ b/core/indexing/docs/suggestions/index.ts
@@ -1,4 +1,5 @@
import {
+ DocsSuggestions,
FilePathAndName,
IDE,
PackageDetails,
@@ -80,7 +81,7 @@ export async function getAllSuggestedDocs(ide: IDE) {
});
// Get documentation links for all packages
- const docsByLanguage: Record<string, PackageDocsResult[]> = {};
+ const docsByLanguage: DocsSuggestions = {};
await Promise.all(
PACKAGE_CRAWLERS.map(async (Crawler) => {
const crawler = new Crawler();
@@ -112,7 +113,6 @@ export async function getAllSuggestedDocs(ide: IDE) {
);
}),
);
-
return docsByLanguage;
}
diff --git a/core/protocol/passThrough.ts b/core/protocol/passThrough.ts
index ebe2a86977..2bd8de2c18 100644
--- a/core/protocol/passThrough.ts
+++ b/core/protocol/passThrough.ts
@@ -75,4 +75,5 @@ export const CORE_TO_WEBVIEW_PASS_THROUGH: (keyof ToWebviewFromCoreProtocol)[] =
"getWebviewHistoryLength",
"signInToControlPlane",
"openDialogMessage",
+ "docs/suggestions",
];
diff --git a/core/protocol/webview.ts b/core/protocol/webview.ts
index 0dcdb047f6..bf90497c8a 100644
--- a/core/protocol/webview.ts
+++ b/core/protocol/webview.ts
@@ -2,6 +2,7 @@ import { ConfigValidationError } from "../config/validation.js";
import type {
ContextItemWithId,
+ DocsSuggestions,
IndexingProgressUpdate,
IndexingStatus,
} from "../index.js";
@@ -25,4 +26,5 @@ export type ToWebviewFromIdeOrCoreProtocol = {
getWebviewHistoryLength: [undefined, number];
signInToControlPlane: [undefined, void];
openDialogMessage: ["account", void];
+ "docs/suggestions": [DocsSuggestions, void];
};
diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx
index 0981948869..c21aca5a6c 100644
--- a/gui/src/components/dialogs/AddDocsDialog.tsx
+++ b/gui/src/components/dialogs/AddDocsDialog.tsx
@@ -5,7 +5,7 @@ import {
} from "@heroicons/react/24/outline";
import { SiteIndexingConfig } from "core";
import { usePostHog } from "posthog-js/react";
-import { useContext, useLayoutEffect, useRef, useState } from "react";
+import { useContext, useLayoutEffect, useMemo, useRef, useState } from "react";
import { useDispatch, useSelector } from "react-redux";
import { Button, HelperText, Input, lightGray, SecondaryButton } from "..";
import { IdeMessengerContext } from "../../context/IdeMessenger";
@@ -15,6 +15,7 @@ import {
} from "../../redux/slices/uiStateSlice";
import { RootState } from "../../redux/store";
import IndexingStatusViewer from "../indexing/IndexingStatus";
+import { C } from "core/autocomplete/constants/AutocompleteLanguageInfo";
function AddDocsDialog() {
const posthog = usePostHog();
@@ -34,6 +35,21 @@ function AddDocsDialog() {
(store: RootState) => store.state.indexing.statuses,
);
+ const docsSuggestions = useSelector(
+ (store: RootState) => store.state.docsSuggestions,
+ );
+
+ const docsByLanguage = useMemo(() => {
+ console.log(docsSuggestions);
+ const languages = Object.keys(docsSuggestions);
+ return languages.map((language) => {
+ return {
+ language,
+ packages: docsSuggestions[language],
+ };
+ });
+ }, [docsSuggestions]);
+
const isFormValid = startUrl && title;
useLayoutEffect(() => {
@@ -118,8 +134,23 @@ function AddDocsDialog() {
return (
<div className="p-4">
<div className="mb-8">
- <h1>Add a documentation site</h1>
-
+ <h1>Add documentation</h1>
+ {docsByLanguage.map(({ language, packages }) => {
+ return (
+ <div key={language}>
+ <h1>{language}</h1>
+ <div>
+ {packages.map((pkg) => {
+ return (
+ <div>
+ <p>{pkg.packageInfo.name}</p>
+ </div>
+ );
+ })}
+ </div>
+ </div>
+ );
+ })}
<p>
Continue pre-indexes many common documentation sites, but if there's
one you don't see in the dropdown, enter the URL here.
diff --git a/gui/src/hooks/useSetup.ts b/gui/src/hooks/useSetup.ts
index f183798890..ecc7a7ac92 100644
--- a/gui/src/hooks/useSetup.ts
+++ b/gui/src/hooks/useSetup.ts
@@ -11,6 +11,7 @@ import {
setInactive,
setSelectedProfileId,
setTTSActive,
+ updateDocsSuggestions,
updateIndexingStatus,
} from "../redux/slices/stateSlice";
import { RootState } from "../redux/store";
@@ -85,6 +86,7 @@ function useSetup(dispatch: Dispatch) {
// ON LOAD
useEffect(() => {
ideMessenger.post("docs/getSuggestedDocs", undefined);
+
// Override persisted state
dispatch(setInactive());
@@ -116,6 +118,10 @@ function useSetup(dispatch: Dispatch) {
}
}, []);
+ useWebviewListener("docs/suggestions", async (data) => {
+ dispatch(updateDocsSuggestions(data));
+ });
+
const { streamResponse } = useChatHandler(dispatch, ideMessenger);
const defaultModelTitle = useSelector(
diff --git a/gui/src/redux/slices/stateSlice.ts b/gui/src/redux/slices/stateSlice.ts
index 770e9ce698..e965455c4f 100644
--- a/gui/src/redux/slices/stateSlice.ts
+++ b/gui/src/redux/slices/stateSlice.ts
@@ -5,6 +5,7 @@ import {
ChatMessage,
Checkpoint,
ContextItemWithId,
+ DocsSuggestions,
FileSymbolMap,
IndexingStatus,
PersistedSessionInfo,
@@ -47,6 +48,7 @@ type State = {
};
streamAborter: AbortController;
isMultifileEdit: boolean;
+ docsSuggestions: DocsSuggestions;
};
const initialState: State = {
@@ -89,6 +91,7 @@ const initialState: State = {
},
},
streamAborter: new AbortController(),
+ docsSuggestions: {},
};
export const stateSlice = createSlice({
@@ -447,6 +450,13 @@ export const stateSlice = createSlice({
[payload.type]: payload.hidden,
};
},
+ updateDocsSuggestions: (
+ state,
+ { payload }: PayloadAction<DocsSuggestions>,
+ ) => {
+ console.log("FROM REDUX", payload);
+ state.docsSuggestions = payload;
+ },
},
});
@@ -480,6 +490,7 @@ export const {
updateIndexingStatus,
setIndexingChatPeekHidden,
abortStream,
+ updateDocsSuggestions,
} = stateSlice.actions;
export default stateSlice.reducer;
From 53f3638b8f0ec703579f3136f7293d4d88ea8a28 Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Thu, 21 Nov 2024 16:37:28 -0800
Subject: [PATCH 04/17] suggested docs continued
---
core/index.d.ts | 11 +++++--
core/indexing/docs/suggestions/index.ts | 30 ++++++++++++-----
.../suggestions/packageCrawlers/Python.ts | 29 +++++++++++++----
.../docs/suggestions/packageCrawlers/TsJs.ts | 24 +++++++++++---
gui/src/components/dialogs/AddDocsDialog.tsx | 32 ++++---------------
gui/src/components/dialogs/SuggestedDocs.tsx | 18 +++++++++++
gui/src/redux/slices/stateSlice.ts | 9 +++---
7 files changed, 100 insertions(+), 53 deletions(-)
create mode 100644 gui/src/components/dialogs/SuggestedDocs.tsx
diff --git a/core/index.d.ts b/core/index.d.ts
index d6b2c1970c..328462e70e 100644
--- a/core/index.d.ts
+++ b/core/index.d.ts
@@ -1196,6 +1196,12 @@ export interface FilePathAndName {
path: string;
name: string;
}
+
+export interface PackageFilePathAndName extends FilePathAndName {
+ language: string; // e.g. javascript
+ registry: string; // e.g. npm
+}
+
export type ParsedPackageInfo = {
language: string;
name: string;
@@ -1207,6 +1213,8 @@ export type PackageDetails = {
docsLink?: string;
title?: string;
description?: string;
+ repo?: string;
+ license?: string;
};
export type PackageDetailsSuccess = PackageDetails & {
@@ -1219,6 +1227,3 @@ export type PackageDocsResult = {
| { error: string; details?: never }
| { details: PackageDetailsSuccess; error?: never }
);
-
-// language -> package -> package info
-export type DocsSuggestions = Record<string, PackageDocsResult[]>;
diff --git a/core/indexing/docs/suggestions/index.ts b/core/indexing/docs/suggestions/index.ts
index 3ee4fb6426..0f86f53902 100644
--- a/core/indexing/docs/suggestions/index.ts
+++ b/core/indexing/docs/suggestions/index.ts
@@ -1,9 +1,9 @@
import {
- DocsSuggestions,
+ PackageDocsResult,
FilePathAndName,
+ PackageFilePathAndName,
IDE,
PackageDetails,
- PackageDocsResult,
ParsedPackageInfo,
} from "../../..";
import { walkDir } from "../../walkDir";
@@ -15,9 +15,9 @@ const PACKAGE_CRAWLERS = [TypeScriptPackageCrawler, PythonPackageCrawler];
export interface PackageCrawler {
language: string;
- getPackageFiles(files: FilePathAndName[]): FilePathAndName[];
+ getPackageFiles(files: FilePathAndName[]): PackageFilePathAndName[];
parsePackageFile(
- file: FilePathAndName,
+ file: PackageFilePathAndName,
contents: string,
): ParsedPackageInfo[];
getPackageDetails(packageInfo: ParsedPackageInfo): Promise<PackageDetails>;
@@ -37,7 +37,7 @@ export async function getAllSuggestedDocs(ide: IDE) {
}));
// Build map of language -> package files
- const packageFilesByLanguage: Record<string, FilePathAndName[]> = {};
+ const packageFilesByLanguage: Record<string, PackageFilePathAndName[]> = {};
for (const Crawler of PACKAGE_CRAWLERS) {
const crawler = new Crawler();
const packageFilePaths = crawler.getPackageFiles(allFiles);
@@ -80,13 +80,26 @@ export async function getAllSuggestedDocs(ide: IDE) {
});
});
+ // Deduplicate packages per language
+ // TODO - this is where you would allow docs for different versions
+ // by e.g. using "name-version" as the map key instead of just name
+ // For now have not allowed
+ const languages = Object.keys(packagesByLanguage);
+ languages.forEach((language) => {
+ const packages = packagesByLanguage[language];
+ const uniquePackages = Array.from(
+ new Map(packages.map((pkg) => [pkg.name, pkg])).values(),
+ );
+ packagesByLanguage[language] = uniquePackages;
+ });
+
// Get documentation links for all packages
- const docsByLanguage: DocsSuggestions = {};
+ const allDocsResults: PackageDocsResult[] = [];
await Promise.all(
PACKAGE_CRAWLERS.map(async (Crawler) => {
const crawler = new Crawler();
const packages = packagesByLanguage[crawler.language];
- docsByLanguage[crawler.language] = await Promise.all(
+ const docsByLanguage = await Promise.all(
packages.map(async (packageInfo) => {
try {
const details = await crawler.getPackageDetails(packageInfo);
@@ -111,9 +124,10 @@ export async function getAllSuggestedDocs(ide: IDE) {
}
}),
);
+ allDocsResults.push(...docsByLanguage);
}),
);
- return docsByLanguage;
+ return allDocsResults;
}
// write me an interface PackageCrawler that contains:
diff --git a/core/indexing/docs/suggestions/packageCrawlers/Python.ts b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
index 233e444285..28b893b66b 100644
--- a/core/indexing/docs/suggestions/packageCrawlers/Python.ts
+++ b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
@@ -2,17 +2,24 @@ import { PackageCrawler } from "..";
import {
FilePathAndName,
PackageDetails,
+ PackageFilePathAndName,
ParsedPackageInfo,
} from "../../../..";
export class PythonPackageCrawler implements PackageCrawler {
language = "python";
- getPackageFiles(files: FilePathAndName[]): FilePathAndName[] {
+ getPackageFiles(files: FilePathAndName[]): PackageFilePathAndName[] {
// For Python, we typically look for files like requirements.txt or Pipfile
- return files.filter(
- (file) => file.name === "requirements.txt" || file.name === "Pipfile",
- );
+ return files
+ .filter(
+ (file) => file.name === "requirements.txt" || file.name === "Pipfile",
+ )
+ .map((file) => ({
+ ...file,
+ language: this.language,
+ registry: "pypi",
+ }));
}
parsePackageFile(
@@ -41,10 +48,18 @@ export class PythonPackageCrawler implements PackageCrawler {
throw new Error(`Could not fetch data for package ${packageInfo.name}`);
}
const data = await response.json();
+ const homePage = data?.info?.home_page as string | undefined;
+
return {
- docsLink: data.info.project_urls?.Documentation as string | undefined,
- // title: data.info.name,
- // description: data.info.summary as string | undefined,
+ docsLink:
+ (data?.info?.project_urls?.Documentation as string | undefined) ??
+ homePage,
+ title: data?.info?.name as string | undefined,
+ description: data?.info?.summary as string | undefined,
+ repo:
+ (data?.info?.project_urls?.Repository as string | undefined) ??
+ homePage,
+ license: data?.info?.license as string | undefined,
};
}
}
diff --git a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
index 107d9ac2a1..c16b8872e8 100644
--- a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
+++ b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
@@ -2,15 +2,22 @@ import { PackageCrawler } from "..";
import {
FilePathAndName,
PackageDetails,
+ PackageFilePathAndName,
ParsedPackageInfo,
} from "../../../..";
export class TypeScriptPackageCrawler implements PackageCrawler {
language = "js-ts";
- getPackageFiles(files: FilePathAndName[]): FilePathAndName[] {
+ getPackageFiles(files: FilePathAndName[]): PackageFilePathAndName[] {
// For Javascript/TypeScript, we look for package.json file
- return files.filter((file) => file.name === "package.json");
+ return files
+ .filter((file) => file.name === "package.json")
+ .map((file) => ({
+ ...file,
+ language: this.language,
+ registry: "npm",
+ }));
}
parsePackageFile(
@@ -22,7 +29,10 @@ export class TypeScriptPackageCrawler implements PackageCrawler {
const dependencies = Object.entries(jsonData.dependencies || {}).concat(
Object.entries(jsonData.devDependencies || {}),
);
- return dependencies.map(([name, version]) => ({
+ const filtered = dependencies.filter(
+ ([name, version]) => !name.startsWith("@types/"),
+ );
+ return filtered.map(([name, version]) => ({
name,
version,
packageFile: file,
@@ -42,8 +52,12 @@ export class TypeScriptPackageCrawler implements PackageCrawler {
const data = await response.json();
return {
docsLink: data.homepage as string | undefined,
- // title: data.name,
- // description: data.description as string | undefined,
+ title: name, // package.json doesn't have specific title field
+ description: data.description as string | undefined,
+ repo: Array.isArray(data.repository)
+ ? (data.respository[0]?.url as string | undefined)
+ : undefined,
+ license: data.license as string | undefined,
};
}
}
diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx
index c21aca5a6c..7dcc478fc6 100644
--- a/gui/src/components/dialogs/AddDocsDialog.tsx
+++ b/gui/src/components/dialogs/AddDocsDialog.tsx
@@ -15,7 +15,6 @@ import {
} from "../../redux/slices/uiStateSlice";
import { RootState } from "../../redux/store";
import IndexingStatusViewer from "../indexing/IndexingStatus";
-import { C } from "core/autocomplete/constants/AutocompleteLanguageInfo";
function AddDocsDialog() {
const posthog = usePostHog();
@@ -39,17 +38,6 @@ function AddDocsDialog() {
(store: RootState) => store.state.docsSuggestions,
);
- const docsByLanguage = useMemo(() => {
- console.log(docsSuggestions);
- const languages = Object.keys(docsSuggestions);
- return languages.map((language) => {
- return {
- language,
- packages: docsSuggestions[language],
- };
- });
- }, [docsSuggestions]);
-
const isFormValid = startUrl && title;
useLayoutEffect(() => {
@@ -135,22 +123,16 @@ function AddDocsDialog() {
<div className="p-4">
<div className="mb-8">
<h1>Add documentation</h1>
- {docsByLanguage.map(({ language, packages }) => {
+ {/* {docsSuggestions.map((docsResult) => {
+ const { error, details } = docsResult;
+ const { language, name, version } = docsResult.packageInfo;
return (
- <div key={language}>
- <h1>{language}</h1>
- <div>
- {packages.map((pkg) => {
- return (
- <div>
- <p>{pkg.packageInfo.name}</p>
- </div>
- );
- })}
- </div>
+ <div key={`${language}-${name}-${version}`}>
+ <p className="m-0 p-0">{`${name}`}</p>
+ <p className="m-0 p-0">{error ? "error" : details.docsLink}</p>
</div>
);
- })}
+ })} */}
<p>
Continue pre-indexes many common documentation sites, but if there's
one you don't see in the dropdown, enter the URL here.
diff --git a/gui/src/components/dialogs/SuggestedDocs.tsx b/gui/src/components/dialogs/SuggestedDocs.tsx
new file mode 100644
index 0000000000..1540ea92f9
--- /dev/null
+++ b/gui/src/components/dialogs/SuggestedDocs.tsx
@@ -0,0 +1,18 @@
+import { PackageDocsResult } from "core";
+
+interface SuggestedDocProps {
+ docResult: PackageDocsResult;
+}
+const SuggestedDoc = ({ docResult }: SuggestedDocProps) => {
+ return <div>SuggestedDoc</div>;
+};
+
+interface SuggestedDocsListProps {
+ docs: PackageDocsResult[];
+ // on
+}
+const SuggestedDocsList = ({ docs }: SuggestedDocsListProps) => {
+ return <div></div>;
+};
+
+export default SuggestedDocsList;
diff --git a/gui/src/redux/slices/stateSlice.ts b/gui/src/redux/slices/stateSlice.ts
index e965455c4f..2feaf80651 100644
--- a/gui/src/redux/slices/stateSlice.ts
+++ b/gui/src/redux/slices/stateSlice.ts
@@ -5,7 +5,7 @@ import {
ChatMessage,
Checkpoint,
ContextItemWithId,
- DocsSuggestions,
+ PackageDocsResult,
FileSymbolMap,
IndexingStatus,
PersistedSessionInfo,
@@ -48,7 +48,7 @@ type State = {
};
streamAborter: AbortController;
isMultifileEdit: boolean;
- docsSuggestions: DocsSuggestions;
+ docsSuggestions: PackageDocsResult[];
};
const initialState: State = {
@@ -91,7 +91,7 @@ const initialState: State = {
},
},
streamAborter: new AbortController(),
- docsSuggestions: {},
+ docsSuggestions: [],
};
export const stateSlice = createSlice({
@@ -452,9 +452,8 @@ export const stateSlice = createSlice({
},
updateDocsSuggestions: (
state,
- { payload }: PayloadAction<DocsSuggestions>,
+ { payload }: PayloadAction<PackageDocsResult[]>,
) => {
- console.log("FROM REDUX", payload);
state.docsSuggestions = payload;
},
},
From 48806f3f0831de8cf67546ef77467548187641cc Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Thu, 21 Nov 2024 18:07:37 -0800
Subject: [PATCH 05/17] docs suggestions p4
---
core/core.ts | 8 +-
core/index.d.ts | 7 +-
core/indexing/docs/suggestions/index.ts | 36 ++--
.../suggestions/packageCrawlers/Python.ts | 9 +-
.../docs/suggestions/packageCrawlers/TsJs.ts | 27 ++-
gui/src/components/dialogs/AddDocsDialog.tsx | 186 ++++++++++++------
gui/src/components/index.ts | 10 +
.../components/indexing/IndexingStatuses.tsx | 27 ++-
gui/tailwind.config.cjs | 11 +-
9 files changed, 199 insertions(+), 122 deletions(-)
diff --git a/core/core.ts b/core/core.ts
index d81df8a379..b2be3478c3 100644
--- a/core/core.ts
+++ b/core/core.ts
@@ -19,6 +19,7 @@ import { ControlPlaneClient } from "./control-plane/client";
import { streamDiffLines } from "./edit/streamDiffLines";
import { CodebaseIndexer, PauseToken } from "./indexing/CodebaseIndexer";
import DocsService from "./indexing/docs/DocsService";
+import { getAllSuggestedDocs } from "./indexing/docs/suggestions";
import { defaultIgnoreFile } from "./indexing/ignore.js";
import Ollama from "./llm/llms/Ollama";
import { createNewPromptFileV2 } from "./promptFiles/v2/createNewPromptFile";
@@ -28,11 +29,7 @@ import { DevDataSqliteDb } from "./util/devdataSqlite";
import { fetchwithRequestOptions } from "./util/fetchWithOptions";
import { GlobalContext } from "./util/GlobalContext";
import historyManager from "./util/history";
-import {
- editConfigJson,
- getConfigJsonPath,
- setupInitialDotContinueDirectory,
-} from "./util/paths";
+import { editConfigJson, setupInitialDotContinueDirectory } from "./util/paths";
import { Telemetry } from "./util/posthog";
import { getSymbolsForManyFiles } from "./util/treeSitter";
import { TTS } from "./util/tts";
@@ -40,7 +37,6 @@ import { TTS } from "./util/tts";
import type { ContextItemId, IDE, IndexingProgressUpdate } from ".";
import type { FromCoreProtocol, ToCoreProtocol } from "./protocol";
import type { IMessenger, Message } from "./util/messenger";
-import { getAllSuggestedDocs } from "./indexing/docs/suggestions";
export class Core {
// implements IMessenger<ToCoreProtocol, FromCoreProtocol>
diff --git a/core/index.d.ts b/core/index.d.ts
index 328462e70e..5541584ebd 100644
--- a/core/index.d.ts
+++ b/core/index.d.ts
@@ -1198,15 +1198,14 @@ export interface FilePathAndName {
}
export interface PackageFilePathAndName extends FilePathAndName {
- language: string; // e.g. javascript
- registry: string; // e.g. npm
+ packageRegistry: string; // e.g. npm, pypi
}
export type ParsedPackageInfo = {
- language: string;
name: string;
+ packageFile: PackageFilePathAndName;
+ language: string;
version: string;
- packageFile: FilePathAndName;
};
export type PackageDetails = {
diff --git a/core/indexing/docs/suggestions/index.ts b/core/indexing/docs/suggestions/index.ts
index 0f86f53902..4e0c2104ac 100644
--- a/core/indexing/docs/suggestions/index.ts
+++ b/core/indexing/docs/suggestions/index.ts
@@ -9,12 +9,12 @@ import {
import { walkDir } from "../../walkDir";
import { PythonPackageCrawler } from "./packageCrawlers/Python";
-import { TypeScriptPackageCrawler } from "./packageCrawlers/TsJs";
+import { NodePackageCrawler } from "./packageCrawlers/TsJs";
-const PACKAGE_CRAWLERS = [TypeScriptPackageCrawler, PythonPackageCrawler];
+const PACKAGE_CRAWLERS = [NodePackageCrawler, PythonPackageCrawler];
export interface PackageCrawler {
- language: string;
+ packageRegistry: string;
getPackageFiles(files: FilePathAndName[]): PackageFilePathAndName[];
parsePackageFile(
file: PackageFilePathAndName,
@@ -37,17 +37,17 @@ export async function getAllSuggestedDocs(ide: IDE) {
}));
// Build map of language -> package files
- const packageFilesByLanguage: Record<string, PackageFilePathAndName[]> = {};
+ const packageFilesByRegistry: Record<string, PackageFilePathAndName[]> = {};
for (const Crawler of PACKAGE_CRAWLERS) {
const crawler = new Crawler();
const packageFilePaths = crawler.getPackageFiles(allFiles);
- packageFilesByLanguage[crawler.language] = packageFilePaths;
+ packageFilesByRegistry[crawler.packageRegistry] = packageFilePaths;
}
// Get file contents for all unique package files
const uniqueFilePaths = Array.from(
new Set(
- Object.values(packageFilesByLanguage).flatMap((files) =>
+ Object.values(packageFilesByRegistry).flatMap((files) =>
files.map((file) => file.path),
),
),
@@ -63,20 +63,20 @@ export async function getAllSuggestedDocs(ide: IDE) {
);
// Parse package files and build map of language -> packages
- const packagesByLanguage: Record<string, ParsedPackageInfo[]> = {};
+ const packagesByRegistry: Record<string, ParsedPackageInfo[]> = {};
PACKAGE_CRAWLERS.forEach((Crawler) => {
const crawler = new Crawler();
- const packageFiles = packageFilesByLanguage[crawler.language];
+ const packageFiles = packageFilesByRegistry[crawler.packageRegistry];
packageFiles.forEach((file) => {
const contents = fileContents.get(file.path);
if (!contents) {
return;
}
const packages = crawler.parsePackageFile(file, contents);
- if (!packagesByLanguage[crawler.language]) {
- packagesByLanguage[crawler.language] = [];
+ if (!packagesByRegistry[crawler.packageRegistry]) {
+ packagesByRegistry[crawler.packageRegistry] = [];
}
- packagesByLanguage[crawler.language].push(...packages);
+ packagesByRegistry[crawler.packageRegistry].push(...packages);
});
});
@@ -84,13 +84,13 @@ export async function getAllSuggestedDocs(ide: IDE) {
// TODO - this is where you would allow docs for different versions
// by e.g. using "name-version" as the map key instead of just name
// For now have not allowed
- const languages = Object.keys(packagesByLanguage);
+ const languages = Object.keys(packagesByRegistry);
languages.forEach((language) => {
- const packages = packagesByLanguage[language];
+ const packages = packagesByRegistry[language];
const uniquePackages = Array.from(
new Map(packages.map((pkg) => [pkg.name, pkg])).values(),
);
- packagesByLanguage[language] = uniquePackages;
+ packagesByRegistry[language] = uniquePackages;
});
// Get documentation links for all packages
@@ -98,8 +98,8 @@ export async function getAllSuggestedDocs(ide: IDE) {
await Promise.all(
PACKAGE_CRAWLERS.map(async (Crawler) => {
const crawler = new Crawler();
- const packages = packagesByLanguage[crawler.language];
- const docsByLanguage = await Promise.all(
+ const packages = packagesByRegistry[crawler.packageRegistry];
+ const docsByRegistry = await Promise.all(
packages.map(async (packageInfo) => {
try {
const details = await crawler.getPackageDetails(packageInfo);
@@ -119,12 +119,12 @@ export async function getAllSuggestedDocs(ide: IDE) {
} catch (error) {
return {
packageInfo,
- error: `Error getting package details for ${name}`,
+ error: `Error getting package details for ${packageInfo.name}`,
};
}
}),
);
- allDocsResults.push(...docsByLanguage);
+ allDocsResults.push(...docsByRegistry);
}),
);
return allDocsResults;
diff --git a/core/indexing/docs/suggestions/packageCrawlers/Python.ts b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
index 28b893b66b..d00045f976 100644
--- a/core/indexing/docs/suggestions/packageCrawlers/Python.ts
+++ b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
@@ -7,7 +7,7 @@ import {
} from "../../../..";
export class PythonPackageCrawler implements PackageCrawler {
- language = "python";
+ packageRegistry = "pypi";
getPackageFiles(files: FilePathAndName[]): PackageFilePathAndName[] {
// For Python, we typically look for files like requirements.txt or Pipfile
@@ -17,13 +17,12 @@ export class PythonPackageCrawler implements PackageCrawler {
)
.map((file) => ({
...file,
- language: this.language,
- registry: "pypi",
+ packageRegistry: "pypi",
}));
}
parsePackageFile(
- file: FilePathAndName,
+ file: PackageFilePathAndName,
contents: string,
): ParsedPackageInfo[] {
// Assume the fileContent is a string from a requirements.txt formatted file
@@ -31,7 +30,7 @@ export class PythonPackageCrawler implements PackageCrawler {
.split("\n")
.map((line) => {
const [name, version] = line.split("==");
- return { name, version, packageFile: file, language: this.language };
+ return { name, version, packageFile: file, language: "python" };
})
.filter((pkg) => pkg.name && pkg.version);
}
diff --git a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
index c16b8872e8..42c5007480 100644
--- a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
+++ b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
@@ -6,8 +6,8 @@ import {
ParsedPackageInfo,
} from "../../../..";
-export class TypeScriptPackageCrawler implements PackageCrawler {
- language = "js-ts";
+export class NodePackageCrawler implements PackageCrawler {
+ packageRegistry = "npm";
getPackageFiles(files: FilePathAndName[]): PackageFilePathAndName[] {
// For Javascript/TypeScript, we look for package.json file
@@ -15,13 +15,12 @@ export class TypeScriptPackageCrawler implements PackageCrawler {
.filter((file) => file.name === "package.json")
.map((file) => ({
...file,
- language: this.language,
- registry: "npm",
+ packageRegistry: this.packageRegistry,
}));
}
parsePackageFile(
- file: FilePathAndName,
+ file: PackageFilePathAndName,
contents: string,
): ParsedPackageInfo[] {
// Parse the package.json content
@@ -29,14 +28,24 @@ export class TypeScriptPackageCrawler implements PackageCrawler {
const dependencies = Object.entries(jsonData.dependencies || {}).concat(
Object.entries(jsonData.devDependencies || {}),
);
- const filtered = dependencies.filter(
- ([name, version]) => !name.startsWith("@types/"),
- );
+
+ // Filter out types packages and check if typescript is present
+ let foundTypes = false;
+ const filtered = dependencies.filter(([name, _]) => {
+ if (name.startsWith("@types/")) {
+ foundTypes = true;
+ return false;
+ }
+ if (name.includes("typescript")) {
+ foundTypes = true;
+ }
+ return true;
+ });
return filtered.map(([name, version]) => ({
name,
version,
packageFile: file,
- language: this.language,
+ language: foundTypes ? "ts" : "js",
}));
}
diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx
index 7dcc478fc6..ca6a8bceb1 100644
--- a/gui/src/components/dialogs/AddDocsDialog.tsx
+++ b/gui/src/components/dialogs/AddDocsDialog.tsx
@@ -2,8 +2,11 @@ import {
CheckCircleIcon,
ChevronDownIcon,
ChevronUpIcon,
+ CodeBracketIcon,
+ InformationCircleIcon,
+ LinkIcon,
} from "@heroicons/react/24/outline";
-import { SiteIndexingConfig } from "core";
+import { PackageDocsResult, SiteIndexingConfig } from "core";
import { usePostHog } from "posthog-js/react";
import { useContext, useLayoutEffect, useMemo, useRef, useState } from "react";
import { useDispatch, useSelector } from "react-redux";
@@ -16,6 +19,8 @@ import {
import { RootState } from "../../redux/store";
import IndexingStatusViewer from "../indexing/IndexingStatus";
+import { ToolTip } from "../gui/Tooltip";
+
function AddDocsDialog() {
const posthog = usePostHog();
const dispatch = useDispatch();
@@ -67,72 +72,125 @@ function AddDocsDialog() {
posthog.capture("add_docs_gui", { url: startUrl });
}
- if (submittedConfig) {
- const status = indexingStatuses[submittedConfig.startUrl];
- return (
- <div className="flex flex-col p-4">
- <div className="flex flex-row items-center gap-2">
- <CheckCircleIcon className="h-8 w-8" />
- <h1>{`Docs added`}</h1>
- </div>
- <div className="flex flex-col gap-1 text-stone-500">
- <p className="m-0 p-0">Title: {submittedConfig.title}</p>
- <p className="m-0 p-0">Start URL: {submittedConfig.startUrl}</p>
- {submittedConfig.rootUrl && (
- <p className="m-0 p-0">Root URL: {submittedConfig.rootUrl}</p>
- )}
- {submittedConfig.maxDepth && (
- <p className="m-0 p-0">Max depth: {submittedConfig.maxDepth}</p>
- )}
- {submittedConfig.faviconUrl && (
- <p className="m-0 p-0">Favicon URL: {submittedConfig.faviconUrl}</p>
- )}
- </div>
- {!!status && (
- <div className="mt-4 flex flex-col divide-x-0 divide-y-2 divide-solid divide-zinc-700">
- <p className="m-0 mb-5 p-0 leading-snug">{`Type "@docs" and select ${submittedConfig.title} to reference these docs once indexing is complete. Check indexing status from the "More" page.`}</p>
- <div className="pt-1">
- <IndexingStatusViewer status={status} />
- </div>
- </div>
- )}
- <div className="mt-4 flex flex-row items-center justify-end gap-4">
- <SecondaryButton
- className=""
- onClick={() => {
- setSubmittedConfig(undefined);
- }}
- >
- Add another
- </SecondaryButton>
- <Button
- className=""
- onClick={() => {
- dispatch(setDialogMessage(undefined));
- dispatch(setShowDialog(false));
- }}
- >
- Done
- </Button>
- </div>
- </div>
- );
- }
+ const handleSelectSuggestion = (docsResult: PackageDocsResult) => {};
+
+ // if (submittedConfig) {
+ // const status = indexingStatuses[submittedConfig.startUrl];
+ // return (
+ // <div className="flex flex-col p-4">
+ // <div className="flex flex-row items-center gap-2">
+ // <CheckCircleIcon className="h-8 w-8" />
+ // <h1>{`Docs added`}</h1>
+ // </div>
+ // <div className="flex flex-col gap-1 text-stone-500">
+ // <p className="m-0 p-0">Title: {submittedConfig.title}</p>
+ // <p className="m-0 p-0">Start URL: {submittedConfig.startUrl}</p>
+ // {submittedConfig.rootUrl && (
+ // <p className="m-0 p-0">Root URL: {submittedConfig.rootUrl}</p>
+ // )}
+ // {submittedConfig.maxDepth && (
+ // <p className="m-0 p-0">Max depth: {submittedConfig.maxDepth}</p>
+ // )}
+ // {submittedConfig.faviconUrl && (
+ // <p className="m-0 p-0">Favicon URL: {submittedConfig.faviconUrl}</p>
+ // )}
+ // </div>
+ // {!!status && (
+ // <div className="mt-4 flex flex-col divide-x-0 divide-y-2 divide-solid divide-zinc-700">
+ // <p className="m-0 mb-5 p-0 leading-snug">{`Type "@docs" and select ${submittedConfig.title} to reference these docs once indexing is complete. Check indexing status from the "More" page.`}</p>
+ // <div className="pt-1">
+ // <IndexingStatusViewer status={status} />
+ // </div>
+ // </div>
+ // )}
+ // <div className="mt-4 flex flex-row items-center justify-end gap-4">
+ // <SecondaryButton
+ // className=""
+ // onClick={() => {
+ // setSubmittedConfig(undefined);
+ // }}
+ // >
+ // Add another
+ // </SecondaryButton>
+ // <Button
+ // className=""
+ // onClick={() => {
+ // dispatch(setDialogMessage(undefined));
+ // dispatch(setShowDialog(false));
+ // }}
+ // >
+ // Done
+ // </Button>
+ // </div>
+ // </div>
+ // );
+ // }
return (
<div className="p-4">
<div className="mb-8">
<h1>Add documentation</h1>
- {/* {docsSuggestions.map((docsResult) => {
- const { error, details } = docsResult;
- const { language, name, version } = docsResult.packageInfo;
- return (
- <div key={`${language}-${name}-${version}`}>
- <p className="m-0 p-0">{`${name}`}</p>
- <p className="m-0 p-0">{error ? "error" : details.docsLink}</p>
- </div>
- );
- })} */}
+ {docsSuggestions.length ? (
+ <div className="no-scrollbar max-h-[300px] overflow-y-auto">
+ <table className="border-collapse p-0">
+ <thead className="bg-vsc-background sticky -top-1 font-bold">
+ <tr className="">
+ <td className="pr-1">
+ <CodeBracketIcon className="h-3.5 w-3.5" />
+ </td>
+ <td className="pr-1">Title</td>
+ {/* <td className="pr-1">Version</td> */}
+ <td className="pr-1">Start Link</td>
+ <td></td>
+ </tr>
+ </thead>
+ <tbody className="p-0">
+ <tr className="whitespace-nowrap">Add docs</tr>
+ {docsSuggestions.map((docsResult) => {
+ const { error, details } = docsResult;
+ const { language, name, version } = docsResult.packageInfo;
+ const id = `${language}-${name}-${version}`;
+ return (
+ <tr key={id} className="p-0">
+ <td>
+ <input type="checkbox"></input>
+ </td>
+ <td>{name}</td>
+ {/* <td>{version}</td> */}
+ <td className="">
+ {error ? (
+ <span className="text-vsc-input-border italic">
+ No docs link found
+ </span>
+ ) : (
+ <span className="flex flex-row items-center gap-2">
+ <div>
+ <LinkIcon className="h-2 w-2" />
+ </div>
+ <p className="lines lines-1 m-0 p-0">
+ {details.docsLink}
+ </p>
+ </span>
+ )}
+ </td>
+ <td>
+ <InformationCircleIcon
+ data-tooltip-id={id}
+ className="text-vsc-foreground-muted h-3.5 w-3.5 cursor-help"
+ />
+
+ <ToolTip id={id} place="bottom">
+ <p className="m-0 p-0">{`Version: ${version}`}</p>
+ <p className="m-0 p-0">{`Found in ${docsResult.packageInfo.packageFile.path}`}</p>
+ </ToolTip>
+ </td>
+ </tr>
+ );
+ })}
+ </tbody>
+ </table>
+ </div>
+ ) : null}
<p>
Continue pre-indexes many common documentation sites, but if there's
one you don't see in the dropdown, enter the URL here.
@@ -143,7 +201,7 @@ function AddDocsDialog() {
so that you can ask questions.
</p>
</div>
-
+ {/*
<form onSubmit={onSubmit} className="flex flex-col space-y-4">
<label>
Title
@@ -219,7 +277,7 @@ function AddDocsDialog() {
Submit
</Button>
</div>
- </form>
+ </form> */}
</div>
);
}
diff --git a/gui/src/components/index.ts b/gui/src/components/index.ts
index e380f3ef6f..74f27a44a3 100644
--- a/gui/src/components/index.ts
+++ b/gui/src/components/index.ts
@@ -4,6 +4,10 @@ import { getFontSize, isJetBrains } from "../util";
export const VSC_INPUT_BACKGROUND_VAR = "--vscode-input-background";
export const VSC_BACKGROUND_VAR = "--vscode-sideBar-background";
export const VSC_FOREGROUND_VAR = "--vscode-editor-foreground";
+export const VSC_FOREGROUND_MUTED_VAR = "--vscode-foreground-muted";
+export const VSC_DESCRIPTION_FOREGROUND = "--vscode-descriptionForeground";
+export const VSC_INPUT_PLACEHOLDER_FOREGROUND =
+ "--vscode-input-placeholderForeground";
export const VSC_BUTTON_BACKGROUND_VAR = "--vscode-button-background";
export const VSC_BUTTON_FOREGROUND_VAR = "--vscode-button-foreground";
export const VSC_EDITOR_BACKGROUND_VAR = "--vscode-editor-background";
@@ -27,6 +31,9 @@ export const VSC_THEME_COLOR_VARS = [
VSC_INPUT_BACKGROUND_VAR,
VSC_BACKGROUND_VAR,
VSC_FOREGROUND_VAR,
+ VSC_FOREGROUND_MUTED_VAR,
+ VSC_DESCRIPTION_FOREGROUND,
+ VSC_INPUT_PLACEHOLDER_FOREGROUND,
VSC_BUTTON_BACKGROUND_VAR,
VSC_BUTTON_FOREGROUND_VAR,
VSC_EDITOR_BACKGROUND_VAR,
@@ -51,6 +58,9 @@ export const vscInputBackground = `var(${VSC_INPUT_BACKGROUND_VAR}, rgb(45 45 45
export const vscQuickInputBackground = `var(${VSC_QUICK_INPUT_BACKGROUND_VAR}, ${VSC_INPUT_BACKGROUND_VAR}, rgb(45 45 45))`;
export const vscBackground = `var(${VSC_BACKGROUND_VAR}, rgb(30 30 30))`;
export const vscForeground = `var(${VSC_FOREGROUND_VAR}, #fff)`;
+export const vscForegroundMuted = `var(${VSC_FOREGROUND_MUTED_VAR}, #999)`;
+export const vscDescriptionForeground = `var(${VSC_DESCRIPTION_FOREGROUND}, #999)`;
+export const vscInputPlaceholderForeground = `var(${VSC_INPUT_PLACEHOLDER_FOREGROUND}, #999)`;
export const vscButtonBackground = `var(${VSC_BUTTON_BACKGROUND_VAR}, #1bbe84)`;
export const vscButtonForeground = `var(${VSC_BUTTON_FOREGROUND_VAR}, #ffffff)`;
export const vscEditorBackground = `var(${VSC_EDITOR_BACKGROUND_VAR}, ${VSC_BACKGROUND_VAR}, rgb(30 30 30))`;
diff --git a/gui/src/components/indexing/IndexingStatuses.tsx b/gui/src/components/indexing/IndexingStatuses.tsx
index 7441212f91..61bf102a98 100644
--- a/gui/src/components/indexing/IndexingStatuses.tsx
+++ b/gui/src/components/indexing/IndexingStatuses.tsx
@@ -34,20 +34,19 @@ function IndexingStatuses() {
Manage your documentation sources
</span>
{/* <div className="flex max-h-[170px] flex-col gap-1 overflow-x-hidden overflow-y-scroll pr-2"> */}
- {docsStatuses.length ? (
- docsStatuses.map((status) => {
- return <IndexingStatusViewer key={status.id} status={status} />;
- })
- ) : (
- <SecondaryButton
- onClick={() => {
- dispatch(setShowDialog(true));
- dispatch(setDialogMessage(<AddDocsDialog />));
- }}
- >
- Add Docs
- </SecondaryButton>
- )}
+ {docsStatuses.length
+ ? docsStatuses.map((status) => {
+ return <IndexingStatusViewer key={status.id} status={status} />;
+ })
+ : null}
+ <SecondaryButton
+ onClick={() => {
+ dispatch(setShowDialog(true));
+ dispatch(setDialogMessage(<AddDocsDialog />));
+ }}
+ >
+ Add Docs
+ </SecondaryButton>
</div>
// </div>
);
diff --git a/gui/tailwind.config.cjs b/gui/tailwind.config.cjs
index f021c748b8..3b3642d6bf 100644
--- a/gui/tailwind.config.cjs
+++ b/gui/tailwind.config.cjs
@@ -46,8 +46,15 @@ module.exports = {
"vsc-badge-background": "var(--vscode-badge-background, #1bbe84)",
"vsc-badge-foreground": "var(--vscode-badge-foreground, #fff)",
"vsc-sidebar-border": "var(--vscode-sideBar-border, transparent)",
- "vsc-find-match": "var(--vscode-editor-findMatchBackground, rgba(255, 255, 0, 0.6))",
- "vsc-find-match-selected": "var(--vscode-editor-findMatchHighlightBackground, rgba(255, 223, 0, 0.8))",
+ "vsc-find-match":
+ "var(--vscode-editor-findMatchBackground, rgba(255, 255, 0, 0.6))",
+ "vsc-find-match-selected":
+ "var(--vscode-editor-findMatchHighlightBackground, rgba(255, 223, 0, 0.8))",
+ "vsc-foreground-muted": "var(--vscode-foreground-muted, #999)",
+ "vsc-description-foreground":
+ "var(--vscode-descriptionForeground, #999)",
+ "vsc-input-placeholder-foreground":
+ "var(--vscode-input-placeholderForeground, #999)",
},
},
},
From 504c0c3843e5388fcd661f9755be9b174e9ef334 Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Fri, 22 Nov 2024 02:51:34 -0800
Subject: [PATCH 06/17] autodetect docs v1
---
core/config/ProfileLifecycleManager.ts | 2 -
core/config/load.ts | 1 +
core/core.ts | 6 +
core/index.d.ts | 2 +
core/indexing/docs/suggestions/index.ts | 39 +-
.../suggestions/packageCrawlers/Python.ts | 2 +-
core/protocol/core.ts | 1 -
core/protocol/webview.ts | 4 +-
gui/src/components/dialogs/AddDocsDialog.tsx | 469 ++++++++++--------
gui/src/components/dialogs/index.tsx | 2 +-
.../components/indexing/ChatIndexingPeeks.tsx | 2 +-
.../components/indexing/DocsIndexingPeeks.tsx | 76 +++
.../components/indexing/IndexingStatuses.tsx | 41 +-
13 files changed, 391 insertions(+), 256 deletions(-)
create mode 100644 gui/src/components/indexing/DocsIndexingPeeks.tsx
diff --git a/core/config/ProfileLifecycleManager.ts b/core/config/ProfileLifecycleManager.ts
index 85116f2133..8d44c8bac9 100644
--- a/core/config/ProfileLifecycleManager.ts
+++ b/core/config/ProfileLifecycleManager.ts
@@ -1,5 +1,3 @@
-import { config } from "dotenv";
-
import {
BrowserSerializedContinueConfig,
ContinueConfig,
diff --git a/core/config/load.ts b/core/config/load.ts
index b692b36bf2..9911c840fe 100644
--- a/core/config/load.ts
+++ b/core/config/load.ts
@@ -527,6 +527,7 @@ function finalToBrowserConfig(
embeddingsProvider: final.embeddingsProvider?.id,
ui: final.ui,
experimental: final.experimental,
+ docs: final.docs,
};
}
diff --git a/core/core.ts b/core/core.ts
index b2be3478c3..331ffbe857 100644
--- a/core/core.ts
+++ b/core/core.ts
@@ -735,6 +735,10 @@ export class Core {
return this.docsService.initStatuses();
});
on("docs/getSuggestedDocs", async (msg) => {
+ if (hasRequestedDocs) {
+ return;
+ } // TODO, remove, hack because of rerendering
+ hasRequestedDocs = true;
const suggestedDocs = await getAllSuggestedDocs(this.ide);
this.messenger.send("docs/suggestions", suggestedDocs);
});
@@ -842,3 +846,5 @@ export class Core {
// private
}
+
+let hasRequestedDocs = false;
diff --git a/core/index.d.ts b/core/index.d.ts
index 5541584ebd..35c072bdb2 100644
--- a/core/index.d.ts
+++ b/core/index.d.ts
@@ -1189,6 +1189,7 @@ export interface BrowserSerializedContinueConfig {
reranker?: RerankerDescription;
experimental?: ExperimentalConfig;
analytics?: AnalyticsConfig;
+ docs?: SiteIndexingConfig[];
}
// DOCS SUGGESTIONS AND PACKAGE INFO
@@ -1210,6 +1211,7 @@ export type ParsedPackageInfo = {
export type PackageDetails = {
docsLink?: string;
+ docsLinkWarning?: string;
title?: string;
description?: string;
repo?: string;
diff --git a/core/indexing/docs/suggestions/index.ts b/core/indexing/docs/suggestions/index.ts
index 4e0c2104ac..db192489ad 100644
--- a/core/indexing/docs/suggestions/index.ts
+++ b/core/indexing/docs/suggestions/index.ts
@@ -63,9 +63,10 @@ export async function getAllSuggestedDocs(ide: IDE) {
);
// Parse package files and build map of language -> packages
- const packagesByRegistry: Record<string, ParsedPackageInfo[]> = {};
+ const packagesByCrawler: Record<string, ParsedPackageInfo[]> = {};
PACKAGE_CRAWLERS.forEach((Crawler) => {
const crawler = new Crawler();
+ packagesByCrawler[crawler.packageRegistry] = [];
const packageFiles = packageFilesByRegistry[crawler.packageRegistry];
packageFiles.forEach((file) => {
const contents = fileContents.get(file.path);
@@ -73,10 +74,7 @@ export async function getAllSuggestedDocs(ide: IDE) {
return;
}
const packages = crawler.parsePackageFile(file, contents);
- if (!packagesByRegistry[crawler.packageRegistry]) {
- packagesByRegistry[crawler.packageRegistry] = [];
- }
- packagesByRegistry[crawler.packageRegistry].push(...packages);
+ packagesByCrawler[crawler.packageRegistry].push(...packages);
});
});
@@ -84,13 +82,13 @@ export async function getAllSuggestedDocs(ide: IDE) {
// TODO - this is where you would allow docs for different versions
// by e.g. using "name-version" as the map key instead of just name
// For now have not allowed
- const languages = Object.keys(packagesByRegistry);
- languages.forEach((language) => {
- const packages = packagesByRegistry[language];
+ const registries = Object.keys(packagesByCrawler);
+ registries.forEach((registry) => {
+ const packages = packagesByCrawler[registry];
const uniquePackages = Array.from(
new Map(packages.map((pkg) => [pkg.name, pkg])).values(),
);
- packagesByRegistry[language] = uniquePackages;
+ packagesByCrawler[registry] = uniquePackages;
});
// Get documentation links for all packages
@@ -98,7 +96,7 @@ export async function getAllSuggestedDocs(ide: IDE) {
await Promise.all(
PACKAGE_CRAWLERS.map(async (Crawler) => {
const crawler = new Crawler();
- const packages = packagesByRegistry[crawler.packageRegistry];
+ const packages = packagesByCrawler[crawler.packageRegistry];
const docsByRegistry = await Promise.all(
packages.map(async (packageInfo) => {
try {
@@ -114,6 +112,11 @@ export async function getAllSuggestedDocs(ide: IDE) {
details: {
...details,
docsLink: details.docsLink,
+ docsLinkWarning: details.docsLink.includes("github.com")
+ ? "Github docs not supported, find the docs site"
+ : details.docsLink.includes("docs")
+ ? undefined
+ : "May not be a docs site, check the URL",
},
};
} catch (error) {
@@ -129,19 +132,3 @@ export async function getAllSuggestedDocs(ide: IDE) {
);
return allDocsResults;
}
-
-// write me an interface PackageCrawler that contains:
-// 1. property `language` to store a given language like "python" or "typescript"
-// 2. has a method `getPackageFiles` which takes a list of file names and decides which ones match package/dependency files (e.g. package.json for typescript, requirements.txt for python, etc)
-// 3. has a method `parsePackageFile` which returns a list of package name and version from a relevant package file, in a standardized format like semver
-// 4. has a method `getDocumentationLink` to check for documentation link for a given package (e.g. GET `https://registry.npmjs.org/<package>` and find docs field for typescript, documentation link in the package metadata for PyPi, etc.)
-// Then, write typescript classes to implement this typescript interface for the languages "python" and "typescript"
-
-// I want to present the user with a list of dependencies and allow them to select which ones to index (embed) documentation for.
-// In order to prevent duplicate file reads, the process will be like this:
-// 1. take in a list of filepaths called `filepaths`
-// 2. loop an array of PackageCrawler classes to build a map of `language` (string) to `packageFilePaths` (string[])
-// 3. Get unique filepaths from `packageFilePaths` and build a map ` of filepath to file contents using an existing `readFile` function, and skipping file reads of already in the map
-// Finally,
-// Add a `` method to the interface and classes that returns
-// Then, assemble the classes in an array, and write a function getAllSuggestedDocs that returns a map of `language` to an ar
diff --git a/core/indexing/docs/suggestions/packageCrawlers/Python.ts b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
index d00045f976..6a486c9849 100644
--- a/core/indexing/docs/suggestions/packageCrawlers/Python.ts
+++ b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
@@ -30,7 +30,7 @@ export class PythonPackageCrawler implements PackageCrawler {
.split("\n")
.map((line) => {
const [name, version] = line.split("==");
- return { name, version, packageFile: file, language: "python" };
+ return { name, version, packageFile: file, language: "py" };
})
.filter((pkg) => pkg.name && pkg.version);
}
diff --git a/core/protocol/core.ts b/core/protocol/core.ts
index 069f9e297a..f5b04c9cf5 100644
--- a/core/protocol/core.ts
+++ b/core/protocol/core.ts
@@ -9,7 +9,6 @@ import type {
DiffLine,
FileSymbolMap,
IdeSettings,
- IndexingStatusMap,
LLMFullCompletionOptions,
MessageContent,
ModelDescription,
diff --git a/core/protocol/webview.ts b/core/protocol/webview.ts
index bf90497c8a..fb6a8837c5 100644
--- a/core/protocol/webview.ts
+++ b/core/protocol/webview.ts
@@ -2,9 +2,9 @@ import { ConfigValidationError } from "../config/validation.js";
import type {
ContextItemWithId,
- DocsSuggestions,
IndexingProgressUpdate,
IndexingStatus,
+ PackageDocsResult,
} from "../index.js";
export type ToWebviewFromIdeOrCoreProtocol = {
@@ -26,5 +26,5 @@ export type ToWebviewFromIdeOrCoreProtocol = {
getWebviewHistoryLength: [undefined, number];
signInToControlPlane: [undefined, void];
openDialogMessage: ["account", void];
- "docs/suggestions": [DocsSuggestions, void];
+ "docs/suggestions": [PackageDocsResult[], void];
};
diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx
index ca6a8bceb1..0a90a60d22 100644
--- a/gui/src/components/dialogs/AddDocsDialog.tsx
+++ b/gui/src/components/dialogs/AddDocsDialog.tsx
@@ -1,12 +1,15 @@
import {
CheckCircleIcon,
+ CheckIcon,
ChevronDownIcon,
ChevronUpIcon,
CodeBracketIcon,
+ ExclamationTriangleIcon,
InformationCircleIcon,
LinkIcon,
+ PlusCircleIcon,
} from "@heroicons/react/24/outline";
-import { PackageDocsResult, SiteIndexingConfig } from "core";
+import { IndexingStatus, PackageDocsResult, SiteIndexingConfig } from "core";
import { usePostHog } from "posthog-js/react";
import { useContext, useLayoutEffect, useMemo, useRef, useState } from "react";
import { useDispatch, useSelector } from "react-redux";
@@ -20,39 +23,79 @@ import { RootState } from "../../redux/store";
import IndexingStatusViewer from "../indexing/IndexingStatus";
import { ToolTip } from "../gui/Tooltip";
+import FileIcon from "../FileIcon";
+import DocsIndexingPeeks from "../indexing/DocsIndexingPeeks";
+import { updateIndexingStatus } from "../../redux/slices/stateSlice";
+import preIndexedDocs from "core/indexing/docs/preIndexedDocs";
function AddDocsDialog() {
const posthog = usePostHog();
const dispatch = useDispatch();
- const ref = useRef<HTMLInputElement>(null);
+ const titleRef = useRef<HTMLInputElement>(null);
+ const urlRef = useRef<HTMLInputElement>(null);
const [title, setTitle] = useState("");
const [startUrl, setStartUrl] = useState("");
const [faviconUrl, setFaviconUrl] = useState("");
- const [isOpen, setIsOpen] = useState(false);
-
- const [submittedConfig, setSubmittedConfig] = useState<SiteIndexingConfig>();
const ideMessenger = useContext(IdeMessengerContext);
const indexingStatuses = useSelector(
(store: RootState) => store.state.indexing.statuses,
);
+ const docsIndexingStatuses: IndexingStatus[] = useMemo(() => {
+ return Object.values(indexingStatuses).filter(
+ (status) => status.type === "docs" && status.status === "indexing",
+ );
+ }, [indexingStatuses]);
+
const docsSuggestions = useSelector(
(store: RootState) => store.state.docsSuggestions,
);
+ const configDocs = useSelector((store: RootState) => store.state.config.docs);
+
+ const sortedDocsSuggestions = useMemo(() => {
+ const docsFromConfig = configDocs ?? [];
+ const filtered = docsSuggestions.filter((sug) => {
+ const startUrl = sug.details?.docsLink;
+ return (
+ !docsFromConfig.find((d) => d.startUrl === startUrl) &&
+ !docsIndexingStatuses.find((s) => s.id === startUrl) &&
+ (startUrl ? !preIndexedDocs[startUrl] : true)
+ );
+ });
+
+ filtered.sort((a, b) => {
+ const rank = (result: PackageDocsResult) => {
+ if (result.error) {
+ return 2;
+ } else if (result.details?.docsLinkWarning) {
+ return 1;
+ } else {
+ return 0;
+ }
+ };
+ return rank(a) - rank(b);
+ });
+ return filtered;
+ }, [docsSuggestions, configDocs, docsIndexingStatuses]);
+
const isFormValid = startUrl && title;
useLayoutEffect(() => {
setTimeout(() => {
- if (ref.current) {
- ref.current.focus();
+ if (titleRef.current) {
+ titleRef.current.focus();
}
}, 100);
- }, [ref]);
+ }, [titleRef]);
+ const closeDialog = () => {
+ dispatch(setShowDialog(false));
+ dispatch(setDialogMessage(undefined));
+ };
function onSubmit(e: any) {
e.preventDefault();
@@ -64,220 +107,236 @@ function AddDocsDialog() {
ideMessenger.post("context/addDocs", siteIndexingConfig);
- setSubmittedConfig(siteIndexingConfig);
setTitle("");
setStartUrl("");
setFaviconUrl("");
posthog.capture("add_docs_gui", { url: startUrl });
+
+ // Optimistic status update
+ dispatch(
+ updateIndexingStatus({
+ type: "docs",
+ description: "Initializing",
+ id: startUrl,
+ embeddingsProviderId: "mock-embeddings-provider-id",
+ progress: 0,
+ status: "indexing",
+ title,
+ url: startUrl,
+ }),
+ );
}
- const handleSelectSuggestion = (docsResult: PackageDocsResult) => {};
+ const handleSelectSuggestion = (docsResult: PackageDocsResult) => {
+ if (docsResult.error) {
+ setTitle(docsResult.packageInfo.name);
+ setStartUrl("");
+ urlRef.current?.focus();
+ return;
+ }
+ if (docsResult.details?.docsLinkWarning) {
+ setTitle(docsResult.packageInfo.name);
+ setStartUrl(docsResult.details.docsLink);
+ urlRef.current?.focus();
+ return;
+ }
+ const siteIndexingConfig: SiteIndexingConfig = {
+ startUrl: docsResult.details.docsLink,
+ title: docsResult.details.title,
+ faviconUrl: undefined,
+ };
+
+ ideMessenger.post("context/addDocs", siteIndexingConfig);
+
+ posthog.capture("add_docs_gui", { url: startUrl });
- // if (submittedConfig) {
- // const status = indexingStatuses[submittedConfig.startUrl];
- // return (
- // <div className="flex flex-col p-4">
- // <div className="flex flex-row items-center gap-2">
- // <CheckCircleIcon className="h-8 w-8" />
- // <h1>{`Docs added`}</h1>
- // </div>
- // <div className="flex flex-col gap-1 text-stone-500">
- // <p className="m-0 p-0">Title: {submittedConfig.title}</p>
- // <p className="m-0 p-0">Start URL: {submittedConfig.startUrl}</p>
- // {submittedConfig.rootUrl && (
- // <p className="m-0 p-0">Root URL: {submittedConfig.rootUrl}</p>
- // )}
- // {submittedConfig.maxDepth && (
- // <p className="m-0 p-0">Max depth: {submittedConfig.maxDepth}</p>
- // )}
- // {submittedConfig.faviconUrl && (
- // <p className="m-0 p-0">Favicon URL: {submittedConfig.faviconUrl}</p>
- // )}
- // </div>
- // {!!status && (
- // <div className="mt-4 flex flex-col divide-x-0 divide-y-2 divide-solid divide-zinc-700">
- // <p className="m-0 mb-5 p-0 leading-snug">{`Type "@docs" and select ${submittedConfig.title} to reference these docs once indexing is complete. Check indexing status from the "More" page.`}</p>
- // <div className="pt-1">
- // <IndexingStatusViewer status={status} />
- // </div>
- // </div>
- // )}
- // <div className="mt-4 flex flex-row items-center justify-end gap-4">
- // <SecondaryButton
- // className=""
- // onClick={() => {
- // setSubmittedConfig(undefined);
- // }}
- // >
- // Add another
- // </SecondaryButton>
- // <Button
- // className=""
- // onClick={() => {
- // dispatch(setDialogMessage(undefined));
- // dispatch(setShowDialog(false));
- // }}
- // >
- // Done
- // </Button>
- // </div>
- // </div>
- // );
- // }
+ // Optimistic status update
+ dispatch(
+ updateIndexingStatus({
+ type: "docs",
+ description: "Initializing",
+ id: docsResult.details.docsLink,
+ embeddingsProviderId: "mock-embeddings-provider-id",
+ progress: 0,
+ status: "indexing",
+ title: docsResult.details.title ?? docsResult.packageInfo.name,
+ url: docsResult.details.docsLink,
+ }),
+ );
+ };
return (
- <div className="p-4">
- <div className="mb-8">
- <h1>Add documentation</h1>
- {docsSuggestions.length ? (
- <div className="no-scrollbar max-h-[300px] overflow-y-auto">
- <table className="border-collapse p-0">
- <thead className="bg-vsc-background sticky -top-1 font-bold">
- <tr className="">
- <td className="pr-1">
- <CodeBracketIcon className="h-3.5 w-3.5" />
- </td>
- <td className="pr-1">Title</td>
- {/* <td className="pr-1">Version</td> */}
- <td className="pr-1">Start Link</td>
- <td></td>
- </tr>
- </thead>
- <tbody className="p-0">
- <tr className="whitespace-nowrap">Add docs</tr>
- {docsSuggestions.map((docsResult) => {
- const { error, details } = docsResult;
- const { language, name, version } = docsResult.packageInfo;
- const id = `${language}-${name}-${version}`;
- return (
- <tr key={id} className="p-0">
- <td>
- <input type="checkbox"></input>
- </td>
- <td>{name}</td>
- {/* <td>{version}</td> */}
- <td className="">
- {error ? (
- <span className="text-vsc-input-border italic">
- No docs link found
- </span>
- ) : (
- <span className="flex flex-row items-center gap-2">
- <div>
- <LinkIcon className="h-2 w-2" />
- </div>
- <p className="lines lines-1 m-0 p-0">
- {details.docsLink}
- </p>
- </span>
- )}
- </td>
- <td>
- <InformationCircleIcon
- data-tooltip-id={id}
- className="text-vsc-foreground-muted h-3.5 w-3.5 cursor-help"
+ <div className="px-2 py-4 sm:px-4">
+ <div className="">
+ <h1 className="mb-0 hidden sm:block">Add documentation</h1>
+ <h1 className="sm:hidden">Add docs</h1>
+ {sortedDocsSuggestions.length && (
+ <p className="m-0 mb-1 mt-4 p-0 font-semibold">Suggestions</p>
+ )}
+ <div className="border-vsc-foreground-muted max-h-[175px] overflow-y-scroll rounded-sm py-1 pr-2">
+ {sortedDocsSuggestions.map((docsResult) => {
+ const { error, details } = docsResult;
+ const { language, name, version } = docsResult.packageInfo;
+ const id = `${language}-${name}-${version}`;
+ return (
+ <>
+ <div
+ key={id}
+ className="grid cursor-pointer grid-cols-[auto_minmax(0,1fr)_minmax(0,1fr)_auto] items-center px-1 py-1 hover:bg-gray-200/10"
+ onClick={() => {
+ handleSelectSuggestion(docsResult);
+ }}
+ >
+ <div className="pr-1">
+ {error ? (
+ <div>
+ <ExclamationTriangleIcon
+ data-tooltip-id={id + "-error"}
+ className="h-4 w-4 text-red-500"
/>
-
- <ToolTip id={id} place="bottom">
- <p className="m-0 p-0">{`Version: ${version}`}</p>
- <p className="m-0 p-0">{`Found in ${docsResult.packageInfo.packageFile.path}`}</p>
+ <ToolTip id={id + "-error"} place="bottom">
+ Docs URL not found
</ToolTip>
- </td>
- </tr>
- );
- })}
- </tbody>
- </table>
- </div>
- ) : null}
- <p>
- Continue pre-indexes many common documentation sites, but if there's
- one you don't see in the dropdown, enter the URL here.
- </p>
-
- <p>
- Continue's indexing engine will crawl the site and generate embeddings
- so that you can ask questions.
- </p>
- </div>
- {/*
- <form onSubmit={onSubmit} className="flex flex-col space-y-4">
- <label>
- Title
- <Input
- type="text"
- placeholder="Title"
- value={title}
- ref={ref}
- onChange={(e) => setTitle(e.target.value)}
- />
- <HelperText>
- The title that will be displayed to users in the `@docs` submenu
- </HelperText>
- </label>
+ </div>
+ ) : details.docsLinkWarning ? (
+ <div>
+ <ExclamationTriangleIcon
+ data-tooltip-id={id + "-warning"}
+ className="h-4 w-4 text-yellow-600"
+ />
+ <ToolTip id={id + "-warning"} place="bottom">
+ Start URL might not lead to docs
+ </ToolTip>
+ </div>
+ ) : (
+ <PlusCircleIcon className="h-4 w-4 text-green-600" />
+ )}
+ </div>
+ <div className="flex items-center gap-0.5">
+ <div className="hidden sm:block">
+ <FileIcon
+ filename={`x.${language}`}
+ height="1rem"
+ width="1rem"
+ />
+ </div>
+ <span className="lines lines-1">{name}</span>
+ </div>
+ <div>
+ {error ? (
+ <span className="text-vsc-input-border italic">
+ No docs link found
+ </span>
+ ) : (
+ <div className="flex items-center gap-2">
+ {/* <div>
+ <LinkIcon className="h-2 w-2" />
+ </div> */}
+ <p className="lines lines-1 m-0 p-0">
+ {details.docsLink}
+ </p>
+ </div>
+ )}
+ </div>
+ <div>
+ <InformationCircleIcon
+ data-tooltip-id={id + "-info"}
+ className="text-vsc-foreground-muted h-3.5 w-3.5 select-none"
+ />
+ <ToolTip id={id + "-info"} place="bottom">
+ <p className="m-0 p-0">{`Version: ${version}`}</p>
+ <p className="m-0 p-0">{`Found in ${docsResult.packageInfo.packageFile.path}`}</p>
+ </ToolTip>
+ </div>
+ </div>
+ <ToolTip id={id} place="bottom">
+ {error ? "Add to form" : "Index these docs"}
+ </ToolTip>
+ </>
+ );
+ })}
+ </div>
+ <div className="mt-3">
+ <form onSubmit={onSubmit} className="flex flex-col gap-1">
+ <div className="flex flex-row gap-2">
+ <label className="flex min-w-16 basis-1/4 flex-col gap-1">
+ <div className="flex flex-row items-center gap-1">
+ <span>Title</span>
+ <div>
+ <InformationCircleIcon
+ data-tooltip-id={"add-docs-form-title"}
+ className="text-vsc-foreground-muted h-3.5 w-3.5 select-none"
+ />
+ <ToolTip id={"add-docs-form-title"} place="top">
+ The title that will be displayed to users in the `@docs`
+ submenu
+ </ToolTip>
+ </div>
+ </div>
- <label>
- Start URL
- <Input
- type="url"
- placeholder="Start URL"
- value={startUrl}
- onChange={(e) => {
- setStartUrl(e.target.value);
- }}
- />
- <HelperText>
- The starting location to begin crawling the documentation site
- </HelperText>
- </label>
+ <Input
+ type="text"
+ placeholder="Title"
+ value={title}
+ ref={titleRef}
+ onChange={(e) => setTitle(e.target.value)}
+ />
+ </label>
- <div
- className="cursor-pointer"
- onClick={() => setIsOpen((prev) => !prev)}
- >
- {isOpen ? (
- <ChevronUpIcon
- width="1.0em"
- height="1.0em"
- style={{ color: lightGray }}
- ></ChevronUpIcon>
- ) : (
- <ChevronDownIcon
- width="1.0em"
- height="1.0em"
- style={{ color: lightGray }}
- ></ChevronDownIcon>
- )}
- <span className="ms-1">Advanced</span>
+ <label className="flex basis-3/4 flex-col gap-1">
+ <div className="flex flex-row items-center gap-1">
+ <span className="lines lines-1 whitespace-nowrap">
+ Start URL
+ </span>
+ <div>
+ <InformationCircleIcon
+ data-tooltip-id={"add-docs-form-url"}
+ className="text-vsc-foreground-muted h-3.5 w-3.5 select-none"
+ />
+ <ToolTip id={"add-docs-form-url"} place="top">
+ The starting location to begin crawling the documentation
+ site
+ </ToolTip>
+ </div>
+ </div>
+ <Input
+ ref={urlRef}
+ type="url"
+ placeholder="Start URL"
+ value={startUrl}
+ onChange={(e) => {
+ setStartUrl(e.target.value);
+ }}
+ />
+ </label>
+ </div>
+ <div className="flex flex-row justify-end gap-2">
+ <SecondaryButton className="min-w-16" onClick={closeDialog}>
+ Done
+ </SecondaryButton>
+ <Button
+ className="min-w-16"
+ disabled={!isFormValid}
+ type="submit"
+ >
+ Go
+ </Button>
+ </div>
+ </form>
</div>
+ </div>
- {isOpen && (
- <div className="pt-2">
- <label>
- Favicon URL [Optional]
- <Input
- type="url"
- placeholder={`${startUrl}/favicon.ico`}
- value={faviconUrl}
- onChange={(e) => {
- setFaviconUrl(e.target.value);
- }}
- />
- <HelperText>
- The URL path to a favicon for the site - by default, it will be
- `/favicon.ico` path from the Start URL
- </HelperText>
- </label>
- </div>
- )}
-
- <div className="flex justify-end">
- <Button disabled={!isFormValid} type="submit">
- Submit
- </Button>
+ <DocsIndexingPeeks statuses={docsIndexingStatuses} />
+ <div className="flex flex-row items-end justify-between gap-2">
+ <div>
+ {docsIndexingStatuses.length ? (
+ <p className="mt-2 p-0 text-xs text-stone-500">
+ It is safe to close this form while indexing
+ </p>
+ ) : null}
</div>
- </form> */}
+ </div>
</div>
);
}
diff --git a/gui/src/components/dialogs/index.tsx b/gui/src/components/dialogs/index.tsx
index 7685109baf..8a3a67fe84 100644
--- a/gui/src/components/dialogs/index.tsx
+++ b/gui/src/components/dialogs/index.tsx
@@ -71,7 +71,7 @@ const TextDialog = (props: TextDialogProps) => {
return (
<ScreenCover onClick={props.onClose} hidden={!props.showDialog}>
<DialogContainer
- className="xs:w-[85%] w-[92%] sm:w-[75%]"
+ className="xs:w-[90%] no-scrollbar max-h-full w-[92%] max-w-[600px] overflow-auto sm:w-[88%] md:w-[80%]"
onClick={(e) => {
e.stopPropagation();
}}
diff --git a/gui/src/components/indexing/ChatIndexingPeeks.tsx b/gui/src/components/indexing/ChatIndexingPeeks.tsx
index c77bdc323d..373689adc2 100644
--- a/gui/src/components/indexing/ChatIndexingPeeks.tsx
+++ b/gui/src/components/indexing/ChatIndexingPeeks.tsx
@@ -104,7 +104,7 @@ function ChatIndexingPeeks() {
return (
<div className="flex flex-col gap-1">
{mergedIndexingStates.map((state) => {
- return <ChatIndexingPeek state={state} />;
+ return <ChatIndexingPeek key={state.type} state={state} />;
})}
</div>
);
diff --git a/gui/src/components/indexing/DocsIndexingPeeks.tsx b/gui/src/components/indexing/DocsIndexingPeeks.tsx
new file mode 100644
index 0000000000..e5b98d42e7
--- /dev/null
+++ b/gui/src/components/indexing/DocsIndexingPeeks.tsx
@@ -0,0 +1,76 @@
+import { useDispatch, useSelector } from "react-redux";
+import { RootState } from "../../redux/store";
+
+import { IndexingStatus } from "core";
+import { useMemo } from "react";
+import { useNavigate } from "react-router-dom";
+import { ArrowPathIcon, EyeSlashIcon } from "@heroicons/react/24/outline";
+import {
+ setDialogMessage,
+ setShowDialog,
+} from "../../redux/slices/uiStateSlice";
+
+export interface DocsIndexingPeekProps {
+ status: IndexingStatus;
+}
+
+function DocsIndexingPeek({ status }: DocsIndexingPeekProps) {
+ const navigate = useNavigate();
+ const dispatch = useDispatch();
+
+ const progressPercentage = useMemo(() => {
+ return Math.min(100, Math.max(0, status.progress * 100));
+ }, [status.progress]);
+
+ return (
+ <div
+ className="flex cursor-pointer flex-row items-center gap-2 rounded-md px-1 text-stone-500 hover:bg-gray-700/10"
+ onClick={() => {
+ navigate("/more");
+ dispatch(setShowDialog(false));
+ dispatch(setDialogMessage(undefined));
+ }}
+ >
+ <p className="m-0 p-0 text-stone-500 group-hover:underline">
+ {status.title}
+ </p>
+ <div className="my-2 h-1.5 flex-1 rounded-md border border-solid border-gray-400">
+ <div
+ className={`h-full rounded-lg bg-stone-500 transition-all duration-200 ease-in-out`}
+ style={{
+ width: `${progressPercentage}%`,
+ }}
+ />
+ </div>
+ <div className="xs:flex hidden flex-row items-center gap-1 text-stone-500">
+ <span className="text-xs no-underline">
+ {progressPercentage.toFixed(0)}%
+ </span>
+ <ArrowPathIcon
+ className={`animate-spin-slow inline-block h-4 w-4 text-stone-500`}
+ ></ArrowPathIcon>
+ </div>
+ </div>
+ );
+}
+
+interface DocsIndexingPeeksProps {
+ statuses: IndexingStatus[];
+}
+
+function DocsIndexingPeekList({ statuses }: DocsIndexingPeeksProps) {
+ if (!statuses.length) return null;
+
+ return (
+ <div className="flex flex-col">
+ <p className="mx-0 my-1.5 p-0 text-stone-500">Currently Indexing</p>
+ <div className="max-h-[100px] overflow-y-auto pr-2">
+ {statuses.map((status) => {
+ return <DocsIndexingPeek key={status.id} status={status} />;
+ })}
+ </div>
+ </div>
+ );
+}
+
+export default DocsIndexingPeekList;
diff --git a/gui/src/components/indexing/IndexingStatuses.tsx b/gui/src/components/indexing/IndexingStatuses.tsx
index 61bf102a98..746bc71ed9 100644
--- a/gui/src/components/indexing/IndexingStatuses.tsx
+++ b/gui/src/components/indexing/IndexingStatuses.tsx
@@ -8,6 +8,8 @@ import {
setShowDialog,
} from "../../redux/slices/uiStateSlice";
import AddDocsDialog from "../dialogs/AddDocsDialog";
+import { PlusCircleIcon } from "@heroicons/react/24/outline";
+import { ToolTip } from "../gui/Tooltip";
function IndexingStatuses() {
const indexingStatuses = useSelector(
@@ -25,30 +27,35 @@ function IndexingStatuses() {
<div className="flex flex-col gap-1">
<div className="flex flex-row items-center justify-between">
<h3 className="mb-1 mt-0 text-xl">@docs indexes</h3>
- {/* <div className="border-1 rounded-full border">
- <ChevronUpIcon className="h-8 w-8" />
- </div> */}
- {/* TODO add some way to hide, scroll, etc. */}
+ {docsStatuses.length ? (
+ <>
+ <PlusCircleIcon
+ data-tooltip-id={"more-add-docs-button"}
+ className="text-vsc-foreground-muted h-5 w-5 cursor-pointer focus:ring-0"
+ onClick={() => {
+ dispatch(setShowDialog(true));
+ dispatch(setDialogMessage(<AddDocsDialog />));
+ }}
+ />
+ <ToolTip id={"more-add-docs-button"} place="top">
+ Add Docs
+ </ToolTip>
+ </>
+ ) : null}
</div>
<span className="text-xs text-stone-500">
Manage your documentation sources
</span>
- {/* <div className="flex max-h-[170px] flex-col gap-1 overflow-x-hidden overflow-y-scroll pr-2"> */}
- {docsStatuses.length
- ? docsStatuses.map((status) => {
+ <div className="flex max-h-[170px] flex-col gap-1 overflow-x-hidden overflow-y-scroll pr-2">
+ {docsStatuses.length ? (
+ docsStatuses.map((status) => {
return <IndexingStatusViewer key={status.id} status={status} />;
})
- : null}
- <SecondaryButton
- onClick={() => {
- dispatch(setShowDialog(true));
- dispatch(setDialogMessage(<AddDocsDialog />));
- }}
- >
- Add Docs
- </SecondaryButton>
+ ) : (
+ <SecondaryButton>Add Docs</SecondaryButton>
+ )}
+ </div>
</div>
- // </div>
);
}
From b41e601798407615b504a2a52ab287d144f4716a Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Fri, 22 Nov 2024 03:05:58 -0800
Subject: [PATCH 07/17] suggested docs UI tweaks
---
.../suggestions/packageCrawlers/Python.ts | 1 -
extensions/vscode/package-lock.json | 2 +-
gui/package-lock.json | 8 +------
gui/src/components/dialogs/AddDocsDialog.tsx | 24 ++++++++++---------
.../components/indexing/DocsIndexingPeeks.tsx | 9 +++----
5 files changed, 20 insertions(+), 24 deletions(-)
diff --git a/core/indexing/docs/suggestions/packageCrawlers/Python.ts b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
index 6a486c9849..3a617568f4 100644
--- a/core/indexing/docs/suggestions/packageCrawlers/Python.ts
+++ b/core/indexing/docs/suggestions/packageCrawlers/Python.ts
@@ -39,7 +39,6 @@ export class PythonPackageCrawler implements PackageCrawler {
packageInfo: ParsedPackageInfo,
): Promise<PackageDetails> {
// Fetch metadata from PyPI to find the documentation link
-
const response = await fetch(
`https://pypi.org/pypi/${packageInfo.name}/json`,
);
diff --git a/extensions/vscode/package-lock.json b/extensions/vscode/package-lock.json
index b30a951dfc..8d680fb94a 100644
--- a/extensions/vscode/package-lock.json
+++ b/extensions/vscode/package-lock.json
@@ -109,7 +109,7 @@
"cheerio": "^1.0.0-rc.12",
"commander": "^12.0.0",
"comment-json": "^4.2.3",
- "dbinfoz": "^0.11.0",
+ "dbinfoz": "^0.14.0",
"diff": "^7.0.0",
"dotenv": "^16.4.5",
"fastest-levenshtein": "^1.0.16",
diff --git a/gui/package-lock.json b/gui/package-lock.json
index 3be234fb8d..1a9e4fe3ff 100644
--- a/gui/package-lock.json
+++ b/gui/package-lock.json
@@ -56,7 +56,6 @@
"socket.io-client": "^4.7.2",
"styled-components": "^5.3.6",
"table": "^6.8.1",
- "tailwind-scrollbar-hide": "^1.1.7",
"tippy.js": "^6.3.7",
"unist-util-visit": "^5.0.0",
"uuid": "^9.0.1",
@@ -112,7 +111,7 @@
"cheerio": "^1.0.0-rc.12",
"commander": "^12.0.0",
"comment-json": "^4.2.3",
- "dbinfoz": "^0.11.0",
+ "dbinfoz": "^0.14.0",
"diff": "^7.0.0",
"dotenv": "^16.4.5",
"fastest-levenshtein": "^1.0.16",
@@ -8984,11 +8983,6 @@
"node": ">=10.0.0"
}
},
- "node_modules/tailwind-scrollbar-hide": {
- "version": "1.1.7",
- "resolved": "https://registry.npmjs.org/tailwind-scrollbar-hide/-/tailwind-scrollbar-hide-1.1.7.tgz",
- "integrity": "sha512-X324n9OtpTmOMqEgDUEA/RgLrNfBF/jwJdctaPZDzB3mppxJk7TLIDmOreEDm1Bq4R9LSPu4Epf8VSdovNU+iA=="
- },
"node_modules/tailwindcss": {
"version": "3.4.12",
"resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.12.tgz",
diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx
index 0a90a60d22..1467a3fa50 100644
--- a/gui/src/components/dialogs/AddDocsDialog.tsx
+++ b/gui/src/components/dialogs/AddDocsDialog.tsx
@@ -167,14 +167,17 @@ function AddDocsDialog() {
};
return (
- <div className="px-2 py-4 sm:px-4">
- <div className="">
+ <div className="px-2 pt-4 sm:px-4">
+ <div className="mb-2">
<h1 className="mb-0 hidden sm:block">Add documentation</h1>
<h1 className="sm:hidden">Add docs</h1>
+ <p className="m-0 mt-2 p-0 text-stone-500">
+ For the @docs context provider
+ </p>
{sortedDocsSuggestions.length && (
<p className="m-0 mb-1 mt-4 p-0 font-semibold">Suggestions</p>
)}
- <div className="border-vsc-foreground-muted max-h-[175px] overflow-y-scroll rounded-sm py-1 pr-2">
+ <div className="border-vsc-foreground-muted max-h-[145px] overflow-y-scroll rounded-sm py-1 pr-2">
{sortedDocsSuggestions.map((docsResult) => {
const { error, details } = docsResult;
const { language, name, version } = docsResult.packageInfo;
@@ -328,14 +331,13 @@ function AddDocsDialog() {
</div>
<DocsIndexingPeeks statuses={docsIndexingStatuses} />
- <div className="flex flex-row items-end justify-between gap-2">
- <div>
- {docsIndexingStatuses.length ? (
- <p className="mt-2 p-0 text-xs text-stone-500">
- It is safe to close this form while indexing
- </p>
- ) : null}
- </div>
+ <div className="flex flex-row items-end justify-start gap-2">
+ {docsIndexingStatuses.length ? (
+ <p className="mt-2 flex flex-row items-center gap-1 p-0 px-1 text-xs text-stone-500">
+ <CheckIcon className="h-3 w-3" />
+ It is safe to close this form while indexing
+ </p>
+ ) : null}
</div>
</div>
);
diff --git a/gui/src/components/indexing/DocsIndexingPeeks.tsx b/gui/src/components/indexing/DocsIndexingPeeks.tsx
index e5b98d42e7..9c5f96ec85 100644
--- a/gui/src/components/indexing/DocsIndexingPeeks.tsx
+++ b/gui/src/components/indexing/DocsIndexingPeeks.tsx
@@ -1,10 +1,9 @@
-import { useDispatch, useSelector } from "react-redux";
-import { RootState } from "../../redux/store";
+import { useDispatch } from "react-redux";
import { IndexingStatus } from "core";
import { useMemo } from "react";
import { useNavigate } from "react-router-dom";
-import { ArrowPathIcon, EyeSlashIcon } from "@heroicons/react/24/outline";
+import { ArrowPathIcon } from "@heroicons/react/24/outline";
import {
setDialogMessage,
setShowDialog,
@@ -63,7 +62,9 @@ function DocsIndexingPeekList({ statuses }: DocsIndexingPeeksProps) {
return (
<div className="flex flex-col">
- <p className="mx-0 my-1.5 p-0 text-stone-500">Currently Indexing</p>
+ <p className="mx-0 my-1.5 p-0 px-1 font-semibold text-stone-500">
+ Currently Indexing
+ </p>
<div className="max-h-[100px] overflow-y-auto pr-2">
{statuses.map((status) => {
return <DocsIndexingPeek key={status.id} status={status} />;
From 6cb9adc110e774e4686f99c1fbe7b1de2e438878 Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Fri, 22 Nov 2024 03:10:45 -0800
Subject: [PATCH 08/17] suggested docs - cleanup
---
gui/src/components/dialogs/SuggestedDocs.tsx | 18 ----
manual-testing-sandbox/package.json | 102 -------------------
manual-testing-sandbox/requirements.txt | 10 --
3 files changed, 130 deletions(-)
delete mode 100644 gui/src/components/dialogs/SuggestedDocs.tsx
delete mode 100644 manual-testing-sandbox/package.json
delete mode 100644 manual-testing-sandbox/requirements.txt
diff --git a/gui/src/components/dialogs/SuggestedDocs.tsx b/gui/src/components/dialogs/SuggestedDocs.tsx
deleted file mode 100644
index 1540ea92f9..0000000000
--- a/gui/src/components/dialogs/SuggestedDocs.tsx
+++ /dev/null
@@ -1,18 +0,0 @@
-import { PackageDocsResult } from "core";
-
-interface SuggestedDocProps {
- docResult: PackageDocsResult;
-}
-const SuggestedDoc = ({ docResult }: SuggestedDocProps) => {
- return <div>SuggestedDoc</div>;
-};
-
-interface SuggestedDocsListProps {
- docs: PackageDocsResult[];
- // on
-}
-const SuggestedDocsList = ({ docs }: SuggestedDocsListProps) => {
- return <div></div>;
-};
-
-export default SuggestedDocsList;
diff --git a/manual-testing-sandbox/package.json b/manual-testing-sandbox/package.json
deleted file mode 100644
index ef77ac910c..0000000000
--- a/manual-testing-sandbox/package.json
+++ /dev/null
@@ -1,102 +0,0 @@
-{
- "name": "gui",
- "private": true,
- "type": "module",
- "author": "Continue Dev, Inc",
- "license": "Apache-2.0",
- "scripts": {
- "dev": "vite",
- "tsc:check": "tsc -p ./ --noEmit",
- "build": "tsc && vite build",
- "preview": "vite preview",
- "test": "vitest run",
- "test:coverage": "vitest run --coverage",
- "test:ui": "vitest --ui",
- "test:watch": "vitest"
- },
- "dependencies": {
- "@headlessui/react": "^1.7.17",
- "@heroicons/react": "^2.0.18",
- "@monaco-editor/react": "^4.6.0",
- "@reduxjs/toolkit": "^1.9.3",
- "@tiptap/core": "^2.3.2",
- "@tiptap/extension-document": "^2.3.2",
- "@tiptap/extension-dropcursor": "^2.1.16",
- "@tiptap/extension-history": "^2.3.2",
- "@tiptap/extension-image": "^2.1.16",
- "@tiptap/extension-mention": "^2.1.13",
- "@tiptap/extension-paragraph": "^2.3.2",
- "@tiptap/extension-placeholder": "^2.1.13",
- "@tiptap/extension-text": "^2.3.2",
- "@tiptap/pm": "^2.1.13",
- "@tiptap/react": "^2.1.13",
- "@tiptap/starter-kit": "^2.1.13",
- "@tiptap/suggestion": "^2.1.13",
- "@types/vscode-webview": "^1.57.1",
- "core": "file:../core",
- "dompurify": "^3.0.6",
- "downshift": "^7.6.0",
- "lodash": "^4.17.21",
- "minisearch": "^7.0.2",
- "onigasm": "^2.2.5",
- "posthog-js": "^1.130.1",
- "prismjs": "^1.29.0",
- "react": "^18.2.0",
- "react-dom": "^18.2.0",
- "react-error-boundary": "^4.0.11",
- "react-hook-form": "^7.47.0",
- "react-intersection-observer": "^9.13.1",
- "react-markdown": "^9.0.1",
- "react-redux": "^8.0.5",
- "react-remark": "^2.1.0",
- "react-router-dom": "^6.14.2",
- "react-switch": "^7.0.0",
- "react-syntax-highlighter": "^15.5.0",
- "react-tooltip": "^5.18.0",
- "redux-persist": "^6.0.0",
- "redux-persist-transform-filter": "^0.0.22",
- "rehype-highlight": "^7.0.0",
- "rehype-katex": "^7.0.0",
- "rehype-wrap-all": "^1.1.0",
- "remark-math": "^6.0.0",
- "reselect": "^5.1.1",
- "seti-file-icons": "^0.0.8",
- "socket.io-client": "^4.7.2",
- "styled-components": "^5.3.6",
- "table": "^6.8.1",
- "tippy.js": "^6.3.7",
- "unist-util-visit": "^5.0.0",
- "uuid": "^9.0.1",
- "vscode-webview": "^1.0.1-beta.1"
- },
- "devDependencies": {
- "@swc/cli": "^0.3.14",
- "@swc/core": "^1.7.26",
- "@testing-library/dom": "^10.4.0",
- "@testing-library/jest-dom": "^6.5.0",
- "@testing-library/react": "^16.0.1",
- "@testing-library/user-event": "^14.5.2",
- "@types/lodash": "^4.17.6",
- "@types/node": "^20.5.6",
- "@types/node-fetch": "^2.6.4",
- "@types/react": "^18.3.11",
- "@types/react-dom": "^18.3.1",
- "@types/react-router-dom": "^5.3.3",
- "@types/react-syntax-highlighter": "^15.5.7",
- "@types/styled-components": "^5.1.26",
- "@vitejs/plugin-react-swc": "^3.7.0",
- "@vitest/coverage-v8": "^2.1.3",
- "@vitest/ui": "^2.1.3",
- "autoprefixer": "^10.4.13",
- "jsdom": "^25.0.1",
- "postcss": "^8.4.21",
- "tailwindcss": "^3.2.7",
- "typescript": "^4.9.3",
- "vite": "^4.1.0",
- "vitest": "^2.1.3"
- },
- "engine-strict": true,
- "engines": {
- "node": ">=20.11.0"
- }
-}
diff --git a/manual-testing-sandbox/requirements.txt b/manual-testing-sandbox/requirements.txt
deleted file mode 100644
index 793d6a8275..0000000000
--- a/manual-testing-sandbox/requirements.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-flask==2.1.1
-requests==2.28.1
-numpy==1.23.4
-pandas==1.5.0
-scipy==1.9.3
-django==4.1.3
-matplotlib==3.6.2
-pytest==7.2.0
-sqlalchemy==1.4.41
-beautifulsoup4==4.11.1
\ No newline at end of file
From 0b0a0e644525f30f7617b95da0347e8042b64f39 Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Fri, 22 Nov 2024 11:07:28 -0800
Subject: [PATCH 09/17] docs suggestions patrick bug fixes p1
---
manual-testing-sandbox/package.json | 45 +++++++++++++++++++
manual-testing-sandbox/requirements.txt | 57 +++++++++++++++++++++++++
2 files changed, 102 insertions(+)
create mode 100644 manual-testing-sandbox/package.json
create mode 100644 manual-testing-sandbox/requirements.txt
diff --git a/manual-testing-sandbox/package.json b/manual-testing-sandbox/package.json
new file mode 100644
index 0000000000..5bcec0c8fc
--- /dev/null
+++ b/manual-testing-sandbox/package.json
@@ -0,0 +1,45 @@
+{
+ "name": "my-react-project",
+ "version": "1.0.0",
+ "description": "A React project using Radix UI for accessible components",
+ "main": "index.js",
+ "scripts": {
+ "start": "react-scripts start",
+ "build": "react-scripts build",
+ "test": "react-scripts test",
+ "eject": "react-scripts eject"
+ },
+ "keywords": [
+ "react",
+ "radix-ui",
+ "accessibility",
+ "frontend"
+ ],
+ "author": "Your Name",
+ "license": "MIT",
+ "dependencies": {
+ "@radix-ui/react-accordion": "^1.0.0",
+ "@radix-ui/react-alert-dialog": "^1.0.0",
+ "@radix-ui/react-checkbox": "^1.0.0",
+ "react": "^18.0.0",
+ "react-dom": "^18.0.0",
+ "react-scripts": "^5.0.0"
+ },
+ "devDependencies": {
+ "@types/react": "^18.0.0",
+ "@types/react-dom": "^18.0.0",
+ "typescript": "^4.5.0"
+ },
+ "browserslist": {
+ "production": [
+ ">0.2%",
+ "not dead",
+ "not op_mini all"
+ ],
+ "development": [
+ "last 1 chrome version",
+ "last 1 firefox version",
+ "last 1 safari version"
+ ]
+ }
+}
diff --git a/manual-testing-sandbox/requirements.txt b/manual-testing-sandbox/requirements.txt
new file mode 100644
index 0000000000..87f2e654bf
--- /dev/null
+++ b/manual-testing-sandbox/requirements.txt
@@ -0,0 +1,57 @@
+# Core packages
+numpy==1.21.2
+pandas==1.3.3
+scipy==1.7.1
+
+# Web development
+flask==2.0.2
+django==3.2.7
+
+# Request handling
+requests==2.26.0
+httpx==0.19.0
+
+# Machine Learning
+scikit-learn==0.24.2
+tensorflow==2.6.0
+
+# Data visualization
+matplotlib==3.4.3
+seaborn==0.11.2
+plotly==5.3.1
+
+# Natural Language Processing
+nltk==3.6.3
+spacy==3.1.2
+
+# Deep Learning
+torch==1.9.1
+keras==2.6.0
+
+# Image processing
+pillow==8.3.2
+opencv-python==4.5.3.56
+
+# Data handling and manipulation
+beautifulsoup4==4.10.0
+sqlalchemy==1.4.25
+xlrd==2.0.1
+openpyxl==3.0.9
+
+# Utilities
+pyyaml==5.4.1
+python-dotenv==0.19.0
+
+# Testing
+pytest==6.2.5
+
+# Logging and Debugging
+loguru==0.5.3
+
+# Asynchronous Programming
+aiohttp==3.7.4
+
+# Others
+jupyter==1.0.0
+gunicorn==20.1.0
+psycopg2==2.9.1
\ No newline at end of file
From f4e9da5828b9f6040728f570692ddfd4f5634e55 Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Fri, 22 Nov 2024 12:19:33 -0800
Subject: [PATCH 10/17] patrick tweaks
---
.../docs/suggestions/packageCrawlers/TsJs.ts | 5 +
gui/src/components/AccountDialog.tsx | 2 -
gui/src/components/dialogs/AddDocsDialog.tsx | 159 +++++++++---------
.../components/indexing/DocsIndexingPeeks.tsx | 20 ++-
.../components/indexing/IndexingStatuses.tsx | 37 ++--
5 files changed, 115 insertions(+), 108 deletions(-)
diff --git a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
index 42c5007480..3a9ef96c2b 100644
--- a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
+++ b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts
@@ -59,6 +59,11 @@ export class NodePackageCrawler implements PackageCrawler {
throw new Error(`Could not fetch data for package ${name}`);
}
const data = await response.json();
+
+ // const dependencies = Object.keys(packageContentData.dependencies || {})
+ // .concat(Object.keys(packageContentData.devDependencies || {}));
+ // const usesTypescript = dependencies.includes("typescript");
+
return {
docsLink: data.homepage as string | undefined,
title: name, // package.json doesn't have specific title field
diff --git a/gui/src/components/AccountDialog.tsx b/gui/src/components/AccountDialog.tsx
index 5438d11df6..db61fe3c0e 100644
--- a/gui/src/components/AccountDialog.tsx
+++ b/gui/src/components/AccountDialog.tsx
@@ -65,8 +65,6 @@ function AccountDialog() {
);
}
- const dispatch = useDispatch();
-
const changeProfileId = (id: string) => {
ideMessenger.post("didChangeSelectedProfile", { id });
dispatch(setSelectedProfileId(id));
diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx
index 1467a3fa50..68039e85c3 100644
--- a/gui/src/components/dialogs/AddDocsDialog.tsx
+++ b/gui/src/components/dialogs/AddDocsDialog.tsx
@@ -7,7 +7,9 @@ import {
ExclamationTriangleIcon,
InformationCircleIcon,
LinkIcon,
+ PencilIcon,
PlusCircleIcon,
+ PlusIcon,
} from "@heroicons/react/24/outline";
import { IndexingStatus, PackageDocsResult, SiteIndexingConfig } from "core";
import { usePostHog } from "posthog-js/react";
@@ -27,6 +29,7 @@ import FileIcon from "../FileIcon";
import DocsIndexingPeeks from "../indexing/DocsIndexingPeeks";
import { updateIndexingStatus } from "../../redux/slices/stateSlice";
import preIndexedDocs from "core/indexing/docs/preIndexedDocs";
+import { NewSessionButton } from "../mainInput/NewSessionButton";
function AddDocsDialog() {
const posthog = usePostHog();
@@ -168,13 +171,13 @@ function AddDocsDialog() {
return (
<div className="px-2 pt-4 sm:px-4">
- <div className="mb-2">
+ <div className="">
<h1 className="mb-0 hidden sm:block">Add documentation</h1>
<h1 className="sm:hidden">Add docs</h1>
<p className="m-0 mt-2 p-0 text-stone-500">
For the @docs context provider
</p>
- {sortedDocsSuggestions.length && (
+ {!!sortedDocsSuggestions.length && (
<p className="m-0 mb-1 mt-4 p-0 font-semibold">Suggestions</p>
)}
<div className="border-vsc-foreground-muted max-h-[145px] overflow-y-scroll rounded-sm py-1 pr-2">
@@ -183,80 +186,65 @@ function AddDocsDialog() {
const { language, name, version } = docsResult.packageInfo;
const id = `${language}-${name}-${version}`;
return (
- <>
- <div
- key={id}
- className="grid cursor-pointer grid-cols-[auto_minmax(0,1fr)_minmax(0,1fr)_auto] items-center px-1 py-1 hover:bg-gray-200/10"
- onClick={() => {
- handleSelectSuggestion(docsResult);
- }}
- >
- <div className="pr-1">
- {error ? (
- <div>
- <ExclamationTriangleIcon
- data-tooltip-id={id + "-error"}
- className="h-4 w-4 text-red-500"
- />
- <ToolTip id={id + "-error"} place="bottom">
- Docs URL not found
- </ToolTip>
- </div>
- ) : details.docsLinkWarning ? (
- <div>
- <ExclamationTriangleIcon
- data-tooltip-id={id + "-warning"}
- className="h-4 w-4 text-yellow-600"
- />
- <ToolTip id={id + "-warning"} place="bottom">
- Start URL might not lead to docs
- </ToolTip>
- </div>
- ) : (
- <PlusCircleIcon className="h-4 w-4 text-green-600" />
- )}
- </div>
- <div className="flex items-center gap-0.5">
- <div className="hidden sm:block">
- <FileIcon
- filename={`x.${language}`}
- height="1rem"
- width="1rem"
+ <div
+ key={id}
+ className="grid cursor-pointer grid-cols-[auto_minmax(0,1fr)_minmax(0,1fr)_auto] items-center px-1 py-1 hover:bg-gray-200/10"
+ onClick={() => {
+ handleSelectSuggestion(docsResult);
+ }}
+ >
+ <div className="pr-1">
+ {error || details.docsLinkWarning ? (
+ <div>
+ <PencilIcon
+ data-tooltip-id={id + "-edit"}
+ className="vsc-foreground-muted h-3 w-3"
/>
+ <ToolTip id={id + "-edit"} place="bottom">
+ This may not be a docs page
+ </ToolTip>
</div>
- <span className="lines lines-1">{name}</span>
+ ) : (
+ <PlusIcon className="text-foreground-muted h-3.5 w-3.5" />
+ )}
+ </div>
+ <div className="flex items-center gap-0.5">
+ <div className="hidden sm:block">
+ <FileIcon
+ filename={`x.${language}`}
+ height="1rem"
+ width="1rem"
+ />
</div>
- <div>
- {error ? (
- <span className="text-vsc-input-border italic">
- No docs link found
- </span>
- ) : (
- <div className="flex items-center gap-2">
- {/* <div>
+ <span className="lines lines-1">{name}</span>
+ </div>
+ <div>
+ {error ? (
+ <span className="text-vsc-foreground-muted italic">
+ No docs link found
+ </span>
+ ) : (
+ <div className="flex items-center gap-2">
+ {/* <div>
<LinkIcon className="h-2 w-2" />
</div> */}
- <p className="lines lines-1 m-0 p-0">
- {details.docsLink}
- </p>
- </div>
- )}
- </div>
- <div>
- <InformationCircleIcon
- data-tooltip-id={id + "-info"}
- className="text-vsc-foreground-muted h-3.5 w-3.5 select-none"
- />
- <ToolTip id={id + "-info"} place="bottom">
- <p className="m-0 p-0">{`Version: ${version}`}</p>
- <p className="m-0 p-0">{`Found in ${docsResult.packageInfo.packageFile.path}`}</p>
- </ToolTip>
- </div>
+ <p className="lines lines-1 m-0 p-0">
+ {details.docsLink}
+ </p>
+ </div>
+ )}
</div>
- <ToolTip id={id} place="bottom">
- {error ? "Add to form" : "Index these docs"}
- </ToolTip>
- </>
+ <div>
+ <InformationCircleIcon
+ data-tooltip-id={id + "-info"}
+ className="text-vsc-foreground-muted h-3.5 w-3.5 select-none"
+ />
+ <ToolTip id={id + "-info"} place="bottom">
+ <p className="m-0 p-0">{`Version: ${version}`}</p>
+ <p className="m-0 p-0">{`Found in ${docsResult.packageInfo.packageFile.path}`}</p>
+ </ToolTip>
+ </div>
+ </div>
);
})}
</div>
@@ -313,31 +301,36 @@ function AddDocsDialog() {
}}
/>
</label>
+ {/* <div>
+ <PlusCircleIcon className="h-5 w-5 self-end" />
+ </div> */}
</div>
<div className="flex flex-row justify-end gap-2">
- <SecondaryButton className="min-w-16" onClick={closeDialog}>
- Done
- </SecondaryButton>
- <Button
+ <SecondaryButton
className="min-w-16"
disabled={!isFormValid}
type="submit"
>
- Go
- </Button>
+ Add
+ </SecondaryButton>
</div>
</form>
</div>
</div>
<DocsIndexingPeeks statuses={docsIndexingStatuses} />
- <div className="flex flex-row items-end justify-start gap-2">
- {docsIndexingStatuses.length ? (
- <p className="mt-2 flex flex-row items-center gap-1 p-0 px-1 text-xs text-stone-500">
- <CheckIcon className="h-3 w-3" />
- It is safe to close this form while indexing
- </p>
- ) : null}
+ <div className="flex flex-row items-end justify-between pb-3">
+ <div>
+ {docsIndexingStatuses.length ? (
+ <p className="mt-2 flex flex-row items-center gap-1 p-0 px-1 text-xs text-stone-500">
+ <CheckIcon className="h-3 w-3" />
+ It is safe to close this form while indexing
+ </p>
+ ) : null}
+ </div>
+ {/* <Button className="min-w-16" onClick={closeDialog}>
+ Done
+ </Button> */}
</div>
</div>
);
diff --git a/gui/src/components/indexing/DocsIndexingPeeks.tsx b/gui/src/components/indexing/DocsIndexingPeeks.tsx
index 9c5f96ec85..e50b4adcf2 100644
--- a/gui/src/components/indexing/DocsIndexingPeeks.tsx
+++ b/gui/src/components/indexing/DocsIndexingPeeks.tsx
@@ -45,9 +45,9 @@ function DocsIndexingPeek({ status }: DocsIndexingPeekProps) {
<span className="text-xs no-underline">
{progressPercentage.toFixed(0)}%
</span>
- <ArrowPathIcon
+ {/* <ArrowPathIcon
className={`animate-spin-slow inline-block h-4 w-4 text-stone-500`}
- ></ArrowPathIcon>
+ ></ArrowPathIcon> */}
</div>
</div>
);
@@ -58,17 +58,19 @@ interface DocsIndexingPeeksProps {
}
function DocsIndexingPeekList({ statuses }: DocsIndexingPeeksProps) {
- if (!statuses.length) return null;
-
return (
- <div className="flex flex-col">
+ <div className="border-vsc-input-border mt-2 flex flex-col border-0 border-t border-solid pt-2">
<p className="mx-0 my-1.5 p-0 px-1 font-semibold text-stone-500">
- Currently Indexing
+ Currently Indexing:
</p>
<div className="max-h-[100px] overflow-y-auto pr-2">
- {statuses.map((status) => {
- return <DocsIndexingPeek key={status.id} status={status} />;
- })}
+ {statuses.length ? (
+ statuses.map((status) => {
+ return <DocsIndexingPeek key={status.id} status={status} />;
+ })
+ ) : (
+ <p className="m-0 pl-1 font-semibold text-stone-500">None</p>
+ )}
</div>
</div>
);
diff --git a/gui/src/components/indexing/IndexingStatuses.tsx b/gui/src/components/indexing/IndexingStatuses.tsx
index 746bc71ed9..3ea04216f4 100644
--- a/gui/src/components/indexing/IndexingStatuses.tsx
+++ b/gui/src/components/indexing/IndexingStatuses.tsx
@@ -28,20 +28,29 @@ function IndexingStatuses() {
<div className="flex flex-row items-center justify-between">
<h3 className="mb-1 mt-0 text-xl">@docs indexes</h3>
{docsStatuses.length ? (
- <>
- <PlusCircleIcon
- data-tooltip-id={"more-add-docs-button"}
- className="text-vsc-foreground-muted h-5 w-5 cursor-pointer focus:ring-0"
- onClick={() => {
- dispatch(setShowDialog(true));
- dispatch(setDialogMessage(<AddDocsDialog />));
- }}
- />
- <ToolTip id={"more-add-docs-button"} place="top">
- Add Docs
- </ToolTip>
- </>
- ) : null}
+ <div
+ className="border-vsc-foreground text-vsc-foreground enabled:hover:bg-vsc-background m-2 rounded border border-solid bg-inherit px-3 py-2 enabled:hover:cursor-pointer enabled:hover:opacity-90 disabled:text-gray-500"
+ onClick={() => {
+ dispatch(setShowDialog(true));
+ dispatch(setDialogMessage(<AddDocsDialog />));
+ }}
+ >
+ Add
+ </div>
+ ) : // <div>
+ // <PlusCircleIcon
+ // data-tooltip-id={"more-add-docs-button"}
+ // className="text-vsc-foreground-muted h-5 w-5 cursor-pointer focus:ring-0"
+ // onClick={() => {
+ // dispatch(setShowDialog(true));
+ // dispatch(setDialogMessage(<AddDocsDialog />));
+ // }}
+ // />
+ // <ToolTip id={"more-add-docs-button"} place="top">
+ // Add Docs
+ // </ToolTip>
+ // </div>
+ null}
</div>
<span className="text-xs text-stone-500">
Manage your documentation sources
From 263c839153c5c9996f4699389d9b2594badf772d Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Fri, 22 Nov 2024 15:41:41 -0800
Subject: [PATCH 11/17] docs - use config.docs instead of inferring from
statuses
---
core/core.ts | 6 +-
core/index.d.ts | 2 -
core/indexing/docs/DocsService.ts | 110 ++++++++++--------
core/protocol/core.ts | 3 +-
core/protocol/passThrough.ts | 2 +-
gui/src/App.tsx | 2 +-
gui/src/components/AccountDialog.tsx | 2 +
gui/src/components/dialogs/AddDocsDialog.tsx | 2 +-
.../components/indexing/DocsIndexingPeeks.tsx | 6 +-
...exingStatus.tsx => DocsIndexingStatus.tsx} | 71 ++++++-----
.../indexing/DocsIndexingStatuses.tsx | 57 +++++++++
.../components/indexing/IndexingStatuses.tsx | 71 -----------
gui/src/hooks/useSetup.ts | 7 +-
gui/src/pages/More/More.tsx | 4 +-
gui/src/redux/slices/stateSlice.ts | 2 +-
.../{ => nested-folder}/package.json | 0
16 files changed, 179 insertions(+), 168 deletions(-)
rename gui/src/components/indexing/{IndexingStatus.tsx => DocsIndexingStatus.tsx} (68%)
create mode 100644 gui/src/components/indexing/DocsIndexingStatuses.tsx
delete mode 100644 gui/src/components/indexing/IndexingStatuses.tsx
rename manual-testing-sandbox/{ => nested-folder}/package.json (100%)
diff --git a/core/core.ts b/core/core.ts
index 331ffbe857..00ceabb4bd 100644
--- a/core/core.ts
+++ b/core/core.ts
@@ -731,9 +731,6 @@ export class Core {
this.docsService.setPaused(msg.data.id, msg.data.paused);
}
});
- on("indexing/initStatuses", async (msg) => {
- return this.docsService.initStatuses();
- });
on("docs/getSuggestedDocs", async (msg) => {
if (hasRequestedDocs) {
return;
@@ -742,6 +739,9 @@ export class Core {
const suggestedDocs = await getAllSuggestedDocs(this.ide);
this.messenger.send("docs/suggestions", suggestedDocs);
});
+ on("docs/initStatuses", async (msg) => {
+ void this.docsService.initStatuses();
+ });
//
on("didChangeSelectedProfile", (msg) => {
diff --git a/core/index.d.ts b/core/index.d.ts
index 583e00a855..30aee52901 100644
--- a/core/index.d.ts
+++ b/core/index.d.ts
@@ -68,8 +68,6 @@ export interface IndexingStatus {
url?: string;
}
-export type IndexingStatusMap = Map<string, IndexingStatus>;
-
export type PromptTemplateFunction = (
history: ChatMessage[],
otherData: Record<string, string>,
diff --git a/core/indexing/docs/DocsService.ts b/core/indexing/docs/DocsService.ts
index 838e8fb542..f9c88cd0ed 100644
--- a/core/indexing/docs/DocsService.ts
+++ b/core/indexing/docs/DocsService.ts
@@ -7,7 +7,6 @@ import {
ContinueConfig,
EmbeddingsProvider,
IDE,
- IndexingStatusMap,
IndexingStatus,
SiteIndexingConfig,
IdeInfo,
@@ -86,7 +85,6 @@ export type AddParams = {
export default class DocsService {
static lanceTableName = "docs";
static sqlitebTableName = "docs";
- static indexingType = "docs";
static preIndexedDocsEmbeddingsProvider =
new TransformersJsEmbeddingsProvider();
@@ -136,38 +134,55 @@ export default class DocsService {
configHandler.onConfigUpdate(this.handleConfigUpdate.bind(this));
}
- readonly statuses: IndexingStatusMap = new Map();
-
- // Function for GUI to retrieve initial pending statuses
- // And kickoff indexing where needed
- async initStatuses() {
- this.config?.docs?.forEach(async (doc) => {
- const currentStatus = this.statuses.get(doc.startUrl);
- if (currentStatus) {
- this.handleStatusUpdate(currentStatus);
- } else {
- this.handleStatusUpdate({
- type: "docs",
- id: doc.startUrl,
- embeddingsProviderId: this.config.embeddingsProvider.id,
- isReindexing: false,
- progress: 0,
- description: "Pending",
- status: "pending",
- title: doc.title,
- debugInfo: `max depth: ${doc.maxDepth}`,
- icon: doc.faviconUrl,
- url: doc.startUrl,
- });
- }
- });
- }
+ readonly statuses: Map<string, IndexingStatus> = new Map();
handleStatusUpdate(update: IndexingStatus) {
this.statuses.set(update.id, update);
this.messenger?.send("indexing/statusUpdate", update);
}
+ // A way for gui to retrieve initial statuses
+ async initStatuses(): Promise<void> {
+ if (!this.config?.docs) {
+ return;
+ }
+ const metadata = await this.listMetadata();
+
+ this.config.docs?.forEach((doc) => {
+ if (!doc.startUrl) {
+ console.error("Invalid config docs entry", doc);
+ return;
+ }
+
+ const sharedStatus = {
+ type: "docs" as IndexingStatus["type"],
+ id: doc.startUrl,
+ embeddingsProviderId: this.config.embeddingsProvider.id,
+ isReindexing: false,
+ title: doc.title,
+ debugInfo: `max depth: ${doc.maxDepth}`,
+ icon: doc.faviconUrl,
+ url: doc.startUrl,
+ };
+ const indexedStatus: IndexingStatus = metadata.find(
+ (meta) => meta.startUrl === doc.startUrl,
+ )
+ ? {
+ ...sharedStatus,
+ progress: 0,
+ description: "Pending",
+ status: "pending",
+ }
+ : {
+ ...sharedStatus,
+ progress: 1,
+ description: "Complete",
+ status: "complete",
+ };
+ this.handleStatusUpdate(indexedStatus);
+ });
+ }
+
abort(startUrl: string) {
const status = this.statuses.get(startUrl);
if (status) {
@@ -362,6 +377,7 @@ export default class DocsService {
const indexExists = await this.hasMetadata(startUrl);
+ // Build status update - most of it is fixed values
const fixedStatus: Pick<
IndexingStatus,
| "type"
@@ -383,6 +399,8 @@ export default class DocsService {
url: siteIndexingConfig.startUrl,
};
+ // Clear current indexes if reIndexing
+ //
if (indexExists) {
if (reIndex) {
await this.deleteIndexes(startUrl);
@@ -398,6 +416,12 @@ export default class DocsService {
}
}
+ // If not preindexed
+ const isPreIndexedDoc = !!preIndexedDocs[siteIndexingConfig.startUrl];
+ if (!isPreIndexedDoc) {
+ this.addToConfig(siteIndexingConfig);
+ }
+
try {
this.handleStatusUpdate({
...fixedStatus,
@@ -435,7 +459,7 @@ export default class DocsService {
articles.push(article);
processedPages++;
- await new Promise((resolve) => setTimeout(resolve, 50)); // Locks down GUI if no sleeping
+ await new Promise((resolve) => setTimeout(resolve, 100)); // Locks down GUI if no sleeping
}
void Telemetry.capture("docs_pages_crawled", {
@@ -788,19 +812,13 @@ export default class DocsService {
}
}
- // Sends "Pending" or current status for each
- await this.initStatuses();
-
- for (const doc of changedDocs) {
- console.log(`Updating indexed doc: ${doc.startUrl}`);
- await this.indexAndAdd(doc, true);
- }
+ await Promise.allSettled([
+ ...changedDocs.map((doc) => this.indexAndAdd(doc, true)),
+ ...newDocs.map((doc) => this.indexAndAdd(doc)),
+ ]);
for (const doc of newDocs) {
- console.log(`Indexing new doc: ${doc.startUrl}`);
void Telemetry.capture("add_docs_config", { url: doc.startUrl });
-
- await this.indexAndAdd(doc);
}
for (const doc of deletedDocs) {
@@ -941,7 +959,7 @@ export default class DocsService {
);
}
- private addToConfig({ siteIndexingConfig }: AddParams) {
+ private addToConfig(siteIndexingConfig: SiteIndexingConfig) {
// Handles the case where a user has manually added the doc to config.json
// so it already exists in the file
const doesDocExist = this.config.docs?.some(
@@ -959,13 +977,6 @@ export default class DocsService {
private async add(params: AddParams) {
await this.addToLance(params);
await this.addMetadataToSqlite(params);
-
- const isPreIndexedDoc =
- !!preIndexedDocs[params.siteIndexingConfig.startUrl];
-
- if (!isPreIndexedDoc) {
- this.addToConfig(params);
- }
}
// Delete methods
@@ -1030,10 +1041,7 @@ export default class DocsService {
console.log(
`Reindexing non-preindexed docs with new embeddings provider: ${embeddingsProvider.id}`,
);
- await this.initStatuses();
- for (const doc of docs) {
- await this.indexAndAdd(doc, true);
- }
+ await Promise.allSettled(docs.map((doc) => this.indexAndAdd(doc)));
// Important that this only is invoked after we have successfully
// cleared and reindex the docs so that the table cannot end up in an
diff --git a/core/protocol/core.ts b/core/protocol/core.ts
index f5b04c9cf5..88cb5642c9 100644
--- a/core/protocol/core.ts
+++ b/core/protocol/core.ts
@@ -9,6 +9,7 @@ import type {
DiffLine,
FileSymbolMap,
IdeSettings,
+ IndexingStatus,
LLMFullCompletionOptions,
MessageContent,
ModelDescription,
@@ -165,8 +166,8 @@ export type ToCoreFromIdeOrWebviewProtocol = {
"indexing/reindex": [{ type: string; id: string }, void];
"indexing/abort": [{ type: string; id: string }, void];
"indexing/setPaused": [{ type: string; id: string; paused: boolean }, void];
- "indexing/initStatuses": [undefined, void];
"docs/getSuggestedDocs": [undefined, void];
+ "docs/initStatuses": [undefined, void];
addAutocompleteModel: [{ model: ModelDescription }, void];
diff --git a/core/protocol/passThrough.ts b/core/protocol/passThrough.ts
index 2bd8de2c18..720a6f66e8 100644
--- a/core/protocol/passThrough.ts
+++ b/core/protocol/passThrough.ts
@@ -48,11 +48,11 @@ export const WEBVIEW_TO_CORE_PASS_THROUGH: (keyof ToCoreFromWebviewProtocol)[] =
"index/forceReIndexFiles",
"index/indexingProgressBarInitialized",
// Docs, etc.
- "indexing/initStatuses",
"indexing/reindex",
"indexing/abort",
"indexing/setPaused",
"docs/getSuggestedDocs",
+ "docs/initStatuses",
//
"completeOnboarding",
"addAutocompleteModel",
diff --git a/gui/src/App.tsx b/gui/src/App.tsx
index 1e34e6e156..ca807a8764 100644
--- a/gui/src/App.tsx
+++ b/gui/src/App.tsx
@@ -8,7 +8,7 @@ import useSubmenuContextProviders from "./hooks/useSubmenuContextProviders";
import { useVscTheme } from "./hooks/useVscTheme";
import { AddNewModel, ConfigureProvider } from "./pages/AddNewModel";
import ConfigErrorPage from "./pages/config-error";
-import Edit from "./pages/Edit";
+import Edit from "./pages/edit";
import ErrorPage from "./pages/error";
import Chat from "./pages/gui";
import History from "./pages/history";
diff --git a/gui/src/components/AccountDialog.tsx b/gui/src/components/AccountDialog.tsx
index db61fe3c0e..5438d11df6 100644
--- a/gui/src/components/AccountDialog.tsx
+++ b/gui/src/components/AccountDialog.tsx
@@ -65,6 +65,8 @@ function AccountDialog() {
);
}
+ const dispatch = useDispatch();
+
const changeProfileId = (id: string) => {
ideMessenger.post("didChangeSelectedProfile", { id });
dispatch(setSelectedProfileId(id));
diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx
index 68039e85c3..b781a5ec8d 100644
--- a/gui/src/components/dialogs/AddDocsDialog.tsx
+++ b/gui/src/components/dialogs/AddDocsDialog.tsx
@@ -22,7 +22,7 @@ import {
setShowDialog,
} from "../../redux/slices/uiStateSlice";
import { RootState } from "../../redux/store";
-import IndexingStatusViewer from "../indexing/IndexingStatus";
+import IndexingStatusViewer from "../indexing/DocsIndexingStatus";
import { ToolTip } from "../gui/Tooltip";
import FileIcon from "../FileIcon";
diff --git a/gui/src/components/indexing/DocsIndexingPeeks.tsx b/gui/src/components/indexing/DocsIndexingPeeks.tsx
index e50b4adcf2..d63012865c 100644
--- a/gui/src/components/indexing/DocsIndexingPeeks.tsx
+++ b/gui/src/components/indexing/DocsIndexingPeeks.tsx
@@ -1,4 +1,4 @@
-import { useDispatch } from "react-redux";
+import { useDispatch, useSelector } from "react-redux";
import { IndexingStatus } from "core";
import { useMemo } from "react";
@@ -8,6 +8,7 @@ import {
setDialogMessage,
setShowDialog,
} from "../../redux/slices/uiStateSlice";
+import { RootState } from "../../redux/store";
export interface DocsIndexingPeekProps {
status: IndexingStatus;
@@ -58,6 +59,9 @@ interface DocsIndexingPeeksProps {
}
function DocsIndexingPeekList({ statuses }: DocsIndexingPeeksProps) {
+ const config = useSelector((store: RootState) => store.state.config);
+ const configDocs = config.docs ?? [];
+ // const docs
return (
<div className="border-vsc-input-border mt-2 flex flex-col border-0 border-t border-solid pt-2">
<p className="mx-0 my-1.5 p-0 px-1 font-semibold text-stone-500">
diff --git a/gui/src/components/indexing/IndexingStatus.tsx b/gui/src/components/indexing/DocsIndexingStatus.tsx
similarity index 68%
rename from gui/src/components/indexing/IndexingStatus.tsx
rename to gui/src/components/indexing/DocsIndexingStatus.tsx
index 6fe72c7419..4fb5013cd4 100644
--- a/gui/src/components/indexing/IndexingStatus.tsx
+++ b/gui/src/components/indexing/DocsIndexingStatus.tsx
@@ -1,6 +1,6 @@
-import { IndexingStatus } from "core";
+import { IndexingStatus, SiteIndexingConfig } from "core";
import { PropsWithChildren, useContext, useMemo } from "react";
-import { useDispatch } from "react-redux";
+import { useDispatch, useSelector } from "react-redux";
import { usePostHog } from "posthog-js/react";
import { IdeMessengerContext } from "../../context/IdeMessenger";
import {
@@ -12,9 +12,10 @@ import {
XMarkIcon,
} from "@heroicons/react/24/outline";
import { updateIndexingStatus } from "../../redux/slices/stateSlice";
+import { RootState } from "../../redux/store";
interface IndexingStatusViewerProps {
- status: IndexingStatus;
+ docConfig: SiteIndexingConfig;
}
const STATUS_TO_ICON: Record<IndexingStatus["status"], any> = {
@@ -27,33 +28,43 @@ const STATUS_TO_ICON: Record<IndexingStatus["status"], any> = {
failed: XMarkIcon, // Since we show an erorr message below
};
-function IndexingStatusViewer({ status }: IndexingStatusViewerProps) {
+function DocsIndexingStatus({ docConfig }: IndexingStatusViewerProps) {
const ideMessenger = useContext(IdeMessengerContext);
const posthog = usePostHog();
const dispatch = useDispatch();
+ // const status = undefined;
+ const status = useSelector(
+ (store: RootState) => store.state.indexing.statuses[docConfig.startUrl],
+ );
+
const reIndex = () =>
ideMessenger.post("indexing/reindex", {
- type: status.type,
- id: status.id,
+ type: "docs",
+ id: docConfig.startUrl,
});
const abort = () => {
ideMessenger.post("indexing/abort", {
- type: status.type,
- id: status.id,
+ type: "docs",
+ id: docConfig.startUrl,
});
// Optimistic abort status
- dispatch(
- updateIndexingStatus({ ...status, status: "aborted", progress: 0 }),
- );
+ if (status) {
+ dispatch(
+ updateIndexingStatus({ ...status, status: "aborted", progress: 0 }),
+ );
+ }
};
const progressPercentage = useMemo(() => {
+ if (!status) {
+ return 0;
+ }
return Math.min(100, Math.max(0, status.progress * 100));
- }, [status.progress]);
+ }, [status?.progress]);
- const Icon = STATUS_TO_ICON[status.status];
+ const Icon = STATUS_TO_ICON[status?.status];
return (
<div className="mt-2 flex w-full flex-col">
@@ -62,24 +73,26 @@ function IndexingStatusViewer({ status }: IndexingStatusViewerProps) {
className={`flex flex-row items-center justify-between gap-2 text-sm`}
>
<div
- className={`flex flex-row items-center gap-2 ${status.url ? "cursor-pointer hover:underline" : ""}`}
+ className={`flex flex-row items-center gap-2 ${status?.url ? "cursor-pointer hover:underline" : ""}`}
onClick={() => {
- if (status.url) {
+ if (status?.url) {
ideMessenger.post("openUrl", status.url);
}
}}
>
- {status.icon ? (
- <img src={status.icon} alt="doc icon" className="h-4 w-4" />
+ {docConfig.faviconUrl ? (
+ <img
+ src={docConfig.faviconUrl}
+ alt="doc icon"
+ className="h-4 w-4"
+ />
) : null}
<p className="lines lines-1 m-0 p-0 text-left">
- {status.title ?? status.id}
+ {docConfig.title ?? docConfig.startUrl}
</p>
- {!!status.url && (
- <ArrowTopRightOnSquareIcon className="mb-0.5 h-3 w-3 text-stone-500" />
- )}
+ <ArrowTopRightOnSquareIcon className="mb-0.5 h-3 w-3 text-stone-500" />
</div>
- {status.status === "pending" ? (
+ {status?.status === "pending" ? (
<div className="text-xs text-stone-500">Pending...</div>
) : (
<div className="flex flex-row items-center gap-1 text-stone-500">
@@ -87,7 +100,7 @@ function IndexingStatusViewer({ status }: IndexingStatusViewerProps) {
{Icon ? (
<Icon
className={`inline-block h-4 w-4 text-stone-500 ${
- status.status === "indexing" ? "animate-spin-slow" : ""
+ status?.status === "indexing" ? "animate-spin-slow" : ""
}`}
></Icon>
) : null}
@@ -98,7 +111,7 @@ function IndexingStatusViewer({ status }: IndexingStatusViewerProps) {
<div className="my-2 h-1.5 w-full rounded-md border border-solid border-gray-400">
<div
className={`h-full rounded-lg transition-all duration-200 ease-in-out ${
- status.status === "failed" ? "bg-red-600" : "bg-stone-500"
+ status?.status === "failed" ? "bg-red-600" : "bg-stone-500"
}`}
style={{
width: `${progressPercentage}%`,
@@ -118,7 +131,7 @@ function IndexingStatusViewer({ status }: IndexingStatusViewerProps) {
paused: () => {},
deleted: () => {},
pending: () => {},
- }[status.status]
+ }[status?.status]
}
>
{
@@ -130,18 +143,16 @@ function IndexingStatusViewer({ status }: IndexingStatusViewerProps) {
paused: "",
deleted: "",
pending: "",
- }[status.status]
+ }[status?.status]
}
</span>
- {/* {status.status === "indexing" && ( */}
<span className="lines lines-1 text-right text-xs text-stone-500">
- {status.description}
+ {status?.description}
</span>
- {/* )} */}
</div>
</div>
);
}
-export default IndexingStatusViewer;
+export default DocsIndexingStatus;
diff --git a/gui/src/components/indexing/DocsIndexingStatuses.tsx b/gui/src/components/indexing/DocsIndexingStatuses.tsx
new file mode 100644
index 0000000000..84fdba9031
--- /dev/null
+++ b/gui/src/components/indexing/DocsIndexingStatuses.tsx
@@ -0,0 +1,57 @@
+import { useMemo } from "react";
+import { useDispatch, useSelector } from "react-redux";
+import { RootState } from "../../redux/store";
+import IndexingStatusViewer from "./DocsIndexingStatus";
+import { SecondaryButton } from "..";
+import {
+ setDialogMessage,
+ setShowDialog,
+} from "../../redux/slices/uiStateSlice";
+import AddDocsDialog from "../dialogs/AddDocsDialog";
+import DocsIndexingStatus from "./DocsIndexingStatus";
+
+function DocsIndexingStatuses() {
+ const dispatch = useDispatch();
+ const config = useSelector((store: RootState) => store.state.config);
+ const configDocs = config.docs ?? [];
+
+ // const indexingStatuses = useSelector(
+ // (store: RootState) => store.state.indexing.statuses,
+ // );
+ // const docsStatuses = useMemo(() => {
+ // const docs = Object.values(indexingStatuses).filter(
+ // (status) => status.type === "docs" && status.status !== "deleted",
+ // );
+ // return docs;
+ // }, [indexingStatuses]);
+
+ return (
+ <div className="flex flex-col gap-1">
+ <div className="flex flex-row items-center justify-between">
+ <h3 className="mb-1 mt-0 text-xl">@docs indexes</h3>
+ {configDocs.length ? (
+ <SecondaryButton
+ className="border-vsc-foreground text-vsc-foreground enabled:hover:bg-vsc-background m-2 rounded border bg-inherit px-3 py-2 enabled:hover:cursor-pointer enabled:hover:opacity-90 disabled:text-gray-500"
+ type="submit"
+ onClick={() => {
+ dispatch(setShowDialog(true));
+ dispatch(setDialogMessage(<AddDocsDialog />));
+ }}
+ >
+ Add
+ </SecondaryButton>
+ ) : null}
+ </div>
+ <span className="text-xs text-stone-500">
+ Manage your documentation sources
+ </span>
+ <div className="flex max-h-[170px] flex-col gap-1 overflow-x-hidden overflow-y-scroll pr-2">
+ {configDocs.map((doc) => {
+ return <DocsIndexingStatus key={doc.startUrl} docConfig={doc} />;
+ })}
+ </div>
+ </div>
+ );
+}
+
+export default DocsIndexingStatuses;
diff --git a/gui/src/components/indexing/IndexingStatuses.tsx b/gui/src/components/indexing/IndexingStatuses.tsx
deleted file mode 100644
index 3ea04216f4..0000000000
--- a/gui/src/components/indexing/IndexingStatuses.tsx
+++ /dev/null
@@ -1,71 +0,0 @@
-import { useMemo } from "react";
-import { useDispatch, useSelector } from "react-redux";
-import { RootState } from "../../redux/store";
-import IndexingStatusViewer from "./IndexingStatus";
-import { Button, SecondaryButton } from "..";
-import {
- setDialogMessage,
- setShowDialog,
-} from "../../redux/slices/uiStateSlice";
-import AddDocsDialog from "../dialogs/AddDocsDialog";
-import { PlusCircleIcon } from "@heroicons/react/24/outline";
-import { ToolTip } from "../gui/Tooltip";
-
-function IndexingStatuses() {
- const indexingStatuses = useSelector(
- (store: RootState) => store.state.indexing.statuses,
- );
- const docsStatuses = useMemo(() => {
- const docs = Object.values(indexingStatuses).filter(
- (status) => status.type === "docs" && status.status !== "deleted",
- );
- return docs;
- }, [indexingStatuses]);
- const dispatch = useDispatch();
-
- return (
- <div className="flex flex-col gap-1">
- <div className="flex flex-row items-center justify-between">
- <h3 className="mb-1 mt-0 text-xl">@docs indexes</h3>
- {docsStatuses.length ? (
- <div
- className="border-vsc-foreground text-vsc-foreground enabled:hover:bg-vsc-background m-2 rounded border border-solid bg-inherit px-3 py-2 enabled:hover:cursor-pointer enabled:hover:opacity-90 disabled:text-gray-500"
- onClick={() => {
- dispatch(setShowDialog(true));
- dispatch(setDialogMessage(<AddDocsDialog />));
- }}
- >
- Add
- </div>
- ) : // <div>
- // <PlusCircleIcon
- // data-tooltip-id={"more-add-docs-button"}
- // className="text-vsc-foreground-muted h-5 w-5 cursor-pointer focus:ring-0"
- // onClick={() => {
- // dispatch(setShowDialog(true));
- // dispatch(setDialogMessage(<AddDocsDialog />));
- // }}
- // />
- // <ToolTip id={"more-add-docs-button"} place="top">
- // Add Docs
- // </ToolTip>
- // </div>
- null}
- </div>
- <span className="text-xs text-stone-500">
- Manage your documentation sources
- </span>
- <div className="flex max-h-[170px] flex-col gap-1 overflow-x-hidden overflow-y-scroll pr-2">
- {docsStatuses.length ? (
- docsStatuses.map((status) => {
- return <IndexingStatusViewer key={status.id} status={status} />;
- })
- ) : (
- <SecondaryButton>Add Docs</SecondaryButton>
- )}
- </div>
- </div>
- );
-}
-
-export default IndexingStatuses;
diff --git a/gui/src/hooks/useSetup.ts b/gui/src/hooks/useSetup.ts
index ecc7a7ac92..34a1e2f748 100644
--- a/gui/src/hooks/useSetup.ts
+++ b/gui/src/hooks/useSetup.ts
@@ -54,9 +54,12 @@ function useSetup(dispatch: Dispatch) {
loadConfig();
const interval = setInterval(() => {
if (initialConfigLoad.current) {
+ // Docs init on config load
+ ideMessenger.post("docs/getSuggestedDocs", undefined);
+ ideMessenger.post("docs/initStatuses", undefined);
+
// This triggers sending pending status to the GUI for relevant docs indexes
clearInterval(interval);
- ideMessenger.post("indexing/initStatuses", undefined);
return;
}
loadConfig();
@@ -85,8 +88,6 @@ function useSetup(dispatch: Dispatch) {
// ON LOAD
useEffect(() => {
- ideMessenger.post("docs/getSuggestedDocs", undefined);
-
// Override persisted state
dispatch(setInactive());
diff --git a/gui/src/pages/More/More.tsx b/gui/src/pages/More/More.tsx
index e743a38343..81648b41b6 100644
--- a/gui/src/pages/More/More.tsx
+++ b/gui/src/pages/More/More.tsx
@@ -15,7 +15,7 @@ import { setOnboardingCard } from "../../redux/slices/uiStateSlice";
import useHistory from "../../hooks/useHistory";
import MoreHelpRow from "./MoreHelpRow";
import IndexingProgress from "./IndexingProgress";
-import IndexingStatuses from "../../components/indexing/IndexingStatuses";
+import DocsIndexingStatuses from "../../components/indexing/DocsIndexingStatuses";
function MorePage() {
useNavigationListener();
@@ -52,7 +52,7 @@ function MorePage() {
<IndexingProgress />
</div>
<div className="flex flex-col py-5">
- <IndexingStatuses />
+ <DocsIndexingStatuses />
</div>
<div className="py-5">
diff --git a/gui/src/redux/slices/stateSlice.ts b/gui/src/redux/slices/stateSlice.ts
index 42fea2a60b..65075d87cd 100644
--- a/gui/src/redux/slices/stateSlice.ts
+++ b/gui/src/redux/slices/stateSlice.ts
@@ -45,7 +45,7 @@ type State = {
nextCodeBlockToApplyIndex: number;
indexing: {
hiddenChatPeekTypes: Record<IndexingStatus["type"], boolean>;
- statuses: Record<string, IndexingStatus>;
+ statuses: Record<string, IndexingStatus>; // status id -> status
};
streamAborter: AbortController;
docsSuggestions: PackageDocsResult[];
diff --git a/manual-testing-sandbox/package.json b/manual-testing-sandbox/nested-folder/package.json
similarity index 100%
rename from manual-testing-sandbox/package.json
rename to manual-testing-sandbox/nested-folder/package.json
From a541fe2f381a5ca3154a22e9ec885841c0d7ca2e Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Fri, 22 Nov 2024 16:03:44 -0800
Subject: [PATCH 12/17] docs suggestions v2
---
gui/src/components/dialogs/AddDocsDialog.tsx | 13 ++-----------
1 file changed, 2 insertions(+), 11 deletions(-)
diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx
index b781a5ec8d..00134a6f73 100644
--- a/gui/src/components/dialogs/AddDocsDialog.tsx
+++ b/gui/src/components/dialogs/AddDocsDialog.tsx
@@ -1,35 +1,25 @@
import {
- CheckCircleIcon,
CheckIcon,
- ChevronDownIcon,
- ChevronUpIcon,
- CodeBracketIcon,
- ExclamationTriangleIcon,
InformationCircleIcon,
- LinkIcon,
PencilIcon,
- PlusCircleIcon,
PlusIcon,
} from "@heroicons/react/24/outline";
import { IndexingStatus, PackageDocsResult, SiteIndexingConfig } from "core";
import { usePostHog } from "posthog-js/react";
import { useContext, useLayoutEffect, useMemo, useRef, useState } from "react";
import { useDispatch, useSelector } from "react-redux";
-import { Button, HelperText, Input, lightGray, SecondaryButton } from "..";
+import { Input, SecondaryButton } from "..";
import { IdeMessengerContext } from "../../context/IdeMessenger";
import {
setDialogMessage,
setShowDialog,
} from "../../redux/slices/uiStateSlice";
import { RootState } from "../../redux/store";
-import IndexingStatusViewer from "../indexing/DocsIndexingStatus";
-
import { ToolTip } from "../gui/Tooltip";
import FileIcon from "../FileIcon";
import DocsIndexingPeeks from "../indexing/DocsIndexingPeeks";
import { updateIndexingStatus } from "../../redux/slices/stateSlice";
import preIndexedDocs from "core/indexing/docs/preIndexedDocs";
-import { NewSessionButton } from "../mainInput/NewSessionButton";
function AddDocsDialog() {
const posthog = usePostHog();
@@ -99,6 +89,7 @@ function AddDocsDialog() {
dispatch(setShowDialog(false));
dispatch(setDialogMessage(undefined));
};
+
function onSubmit(e: any) {
e.preventDefault();
From de984454cd4978cce51ce13a5539a0984b6162b7 Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Mon, 25 Nov 2024 12:25:20 -0800
Subject: [PATCH 13/17] status race condition fix
---
core/indexing/docs/DocsService.ts | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/core/indexing/docs/DocsService.ts b/core/indexing/docs/DocsService.ts
index f9c88cd0ed..0f3724d61d 100644
--- a/core/indexing/docs/DocsService.ts
+++ b/core/indexing/docs/DocsService.ts
@@ -150,10 +150,15 @@ export default class DocsService {
this.config.docs?.forEach((doc) => {
if (!doc.startUrl) {
- console.error("Invalid config docs entry", doc);
+ console.error("Invalid config docs entry, no start");
return;
}
+ const currentStatus = this.statuses.get(doc.startUrl);
+ if (currentStatus) {
+ return currentStatus;
+ }
+
const sharedStatus = {
type: "docs" as IndexingStatus["type"],
id: doc.startUrl,
From d26f82a3fe11c2aac005f4a8deaa81b2cdc37b74 Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Mon, 25 Nov 2024 16:47:54 -0800
Subject: [PATCH 14/17] edit casing bug
---
gui/src/App.tsx | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gui/src/App.tsx b/gui/src/App.tsx
index ca807a8764..82433e2052 100644
--- a/gui/src/App.tsx
+++ b/gui/src/App.tsx
@@ -8,7 +8,7 @@ import useSubmenuContextProviders from "./hooks/useSubmenuContextProviders";
import { useVscTheme } from "./hooks/useVscTheme";
import { AddNewModel, ConfigureProvider } from "./pages/AddNewModel";
import ConfigErrorPage from "./pages/config-error";
-import Edit from "./pages/edit";
+import Edit from "./pages/edit/Edit";
import ErrorPage from "./pages/error";
import Chat from "./pages/gui";
import History from "./pages/history";
From 800e65e51ccc0c978a44bc10de0dedb7b35516dd Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Mon, 25 Nov 2024 17:01:16 -0800
Subject: [PATCH 15/17] tiny tiny
---
gui/src/App.tsx | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gui/src/App.tsx b/gui/src/App.tsx
index 82433e2052..ca807a8764 100644
--- a/gui/src/App.tsx
+++ b/gui/src/App.tsx
@@ -8,7 +8,7 @@ import useSubmenuContextProviders from "./hooks/useSubmenuContextProviders";
import { useVscTheme } from "./hooks/useVscTheme";
import { AddNewModel, ConfigureProvider } from "./pages/AddNewModel";
import ConfigErrorPage from "./pages/config-error";
-import Edit from "./pages/edit/Edit";
+import Edit from "./pages/edit";
import ErrorPage from "./pages/error";
import Chat from "./pages/gui";
import History from "./pages/history";
From 7f5ec0dfc10f9151c8292e5f53d29692ebb7b0df Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Mon, 25 Nov 2024 17:36:16 -0800
Subject: [PATCH 16/17] short
---
gui/src/pages/{Edit => edi}/AddFileButton.tsx | 0
gui/src/pages/{Edit => edi}/CodeToEdit.tsx | 0
gui/src/pages/{Edit => edi}/CodeToEditListItem.tsx | 0
gui/src/pages/{Edit => edi}/Edit.tsx | 0
gui/src/pages/{Edit => edi}/getMultifileEditPrompt.ts | 0
gui/src/pages/{Edit => edi}/index.tsx | 0
6 files changed, 0 insertions(+), 0 deletions(-)
rename gui/src/pages/{Edit => edi}/AddFileButton.tsx (100%)
rename gui/src/pages/{Edit => edi}/CodeToEdit.tsx (100%)
rename gui/src/pages/{Edit => edi}/CodeToEditListItem.tsx (100%)
rename gui/src/pages/{Edit => edi}/Edit.tsx (100%)
rename gui/src/pages/{Edit => edi}/getMultifileEditPrompt.ts (100%)
rename gui/src/pages/{Edit => edi}/index.tsx (100%)
diff --git a/gui/src/pages/Edit/AddFileButton.tsx b/gui/src/pages/edi/AddFileButton.tsx
similarity index 100%
rename from gui/src/pages/Edit/AddFileButton.tsx
rename to gui/src/pages/edi/AddFileButton.tsx
diff --git a/gui/src/pages/Edit/CodeToEdit.tsx b/gui/src/pages/edi/CodeToEdit.tsx
similarity index 100%
rename from gui/src/pages/Edit/CodeToEdit.tsx
rename to gui/src/pages/edi/CodeToEdit.tsx
diff --git a/gui/src/pages/Edit/CodeToEditListItem.tsx b/gui/src/pages/edi/CodeToEditListItem.tsx
similarity index 100%
rename from gui/src/pages/Edit/CodeToEditListItem.tsx
rename to gui/src/pages/edi/CodeToEditListItem.tsx
diff --git a/gui/src/pages/Edit/Edit.tsx b/gui/src/pages/edi/Edit.tsx
similarity index 100%
rename from gui/src/pages/Edit/Edit.tsx
rename to gui/src/pages/edi/Edit.tsx
diff --git a/gui/src/pages/Edit/getMultifileEditPrompt.ts b/gui/src/pages/edi/getMultifileEditPrompt.ts
similarity index 100%
rename from gui/src/pages/Edit/getMultifileEditPrompt.ts
rename to gui/src/pages/edi/getMultifileEditPrompt.ts
diff --git a/gui/src/pages/Edit/index.tsx b/gui/src/pages/edi/index.tsx
similarity index 100%
rename from gui/src/pages/Edit/index.tsx
rename to gui/src/pages/edi/index.tsx
From f7ce697b3f27d2cf88dc67987c94b1fba75b2fb5 Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Mon, 25 Nov 2024 17:36:32 -0800
Subject: [PATCH 17/17] long
---
gui/src/pages/{edi => edit}/AddFileButton.tsx | 0
gui/src/pages/{edi => edit}/CodeToEdit.tsx | 0
gui/src/pages/{edi => edit}/CodeToEditListItem.tsx | 0
gui/src/pages/{edi => edit}/Edit.tsx | 0
gui/src/pages/{edi => edit}/getMultifileEditPrompt.ts | 0
gui/src/pages/{edi => edit}/index.tsx | 0
6 files changed, 0 insertions(+), 0 deletions(-)
rename gui/src/pages/{edi => edit}/AddFileButton.tsx (100%)
rename gui/src/pages/{edi => edit}/CodeToEdit.tsx (100%)
rename gui/src/pages/{edi => edit}/CodeToEditListItem.tsx (100%)
rename gui/src/pages/{edi => edit}/Edit.tsx (100%)
rename gui/src/pages/{edi => edit}/getMultifileEditPrompt.ts (100%)
rename gui/src/pages/{edi => edit}/index.tsx (100%)
diff --git a/gui/src/pages/edi/AddFileButton.tsx b/gui/src/pages/edit/AddFileButton.tsx
similarity index 100%
rename from gui/src/pages/edi/AddFileButton.tsx
rename to gui/src/pages/edit/AddFileButton.tsx
diff --git a/gui/src/pages/edi/CodeToEdit.tsx b/gui/src/pages/edit/CodeToEdit.tsx
similarity index 100%
rename from gui/src/pages/edi/CodeToEdit.tsx
rename to gui/src/pages/edit/CodeToEdit.tsx
diff --git a/gui/src/pages/edi/CodeToEditListItem.tsx b/gui/src/pages/edit/CodeToEditListItem.tsx
similarity index 100%
rename from gui/src/pages/edi/CodeToEditListItem.tsx
rename to gui/src/pages/edit/CodeToEditListItem.tsx
diff --git a/gui/src/pages/edi/Edit.tsx b/gui/src/pages/edit/Edit.tsx
similarity index 100%
rename from gui/src/pages/edi/Edit.tsx
rename to gui/src/pages/edit/Edit.tsx
diff --git a/gui/src/pages/edi/getMultifileEditPrompt.ts b/gui/src/pages/edit/getMultifileEditPrompt.ts
similarity index 100%
rename from gui/src/pages/edi/getMultifileEditPrompt.ts
rename to gui/src/pages/edit/getMultifileEditPrompt.ts
diff --git a/gui/src/pages/edi/index.tsx b/gui/src/pages/edit/index.tsx
similarity index 100%
rename from gui/src/pages/edi/index.tsx
rename to gui/src/pages/edit/index.tsx