From cb071a74194f656a462f53ed503ba725c31958e3 Mon Sep 17 00:00:00 2001 From: Dallin Romney <dallinromney@gmail.com> Date: Thu, 21 Nov 2024 00:42:09 -0800 Subject: [PATCH 01/17] package file parsers --- core/core.ts | 4 + core/indexing/docs/suggestions/index.ts | 100 +++++++++++++++++ .../suggestions/packageCrawlers/Python.ts | 33 ++++++ .../docs/suggestions/packageCrawlers/TsJs.ts | 33 ++++++ core/protocol/core.ts | 1 + core/protocol/passThrough.ts | 1 + extensions/vscode/package-lock.json | 4 +- gui/src/hooks/useSetup.ts | 3 +- manual-testing-sandbox/package.json | 102 ++++++++++++++++++ manual-testing-sandbox/requirements.txt | 10 ++ 10 files changed, 287 insertions(+), 4 deletions(-) create mode 100644 core/indexing/docs/suggestions/index.ts create mode 100644 core/indexing/docs/suggestions/packageCrawlers/Python.ts create mode 100644 core/indexing/docs/suggestions/packageCrawlers/TsJs.ts create mode 100644 manual-testing-sandbox/package.json create mode 100644 manual-testing-sandbox/requirements.txt diff --git a/core/core.ts b/core/core.ts index c6b336c7d0..d080ec34b0 100644 --- a/core/core.ts +++ b/core/core.ts @@ -40,6 +40,7 @@ import { TTS } from "./util/tts"; import type { ContextItemId, IDE, IndexingProgressUpdate } from "."; import type { FromCoreProtocol, ToCoreProtocol } from "./protocol"; import type { IMessenger, Message } from "./util/messenger"; +import { getAllSuggestedDocs } from "./indexing/docs/suggestions"; export class Core { // implements IMessenger<ToCoreProtocol, FromCoreProtocol> @@ -737,6 +738,9 @@ export class Core { on("indexing/initStatuses", async (msg) => { return this.docsService.initStatuses(); }); + on("docs/getSuggestedDocs", async (msg) => { + getAllSuggestedDocs(this.ide); + }); // on("didChangeSelectedProfile", (msg) => { diff --git a/core/indexing/docs/suggestions/index.ts b/core/indexing/docs/suggestions/index.ts new file mode 100644 index 0000000000..d3d68aca29 --- /dev/null +++ b/core/indexing/docs/suggestions/index.ts @@ -0,0 +1,100 @@ +// write me an interface PackageCrawler that contains: +// 1. property `language` to store a given language like "python" or "typescript" +// 2. has a method `getPackageFiles` which takes a list of file names and decides which ones match package/dependency files (e.g. package.json for typescript, requirements.txt for python, etc) +// 3. has a method `parsePackageFile` which returns a list of package name and version from a relevant package file, in a standardized format like semver +// 4. has a method `getDocumentationLink` to check for documentation link for a given package (e.g. GET `https://registry.npmjs.org/<package>` and find docs field for typescript, documentation link in the package metadata for PyPi, etc.) +// Then, write typescript classes to implement this typescript interface for the languages "python" and "typescript" + +import { IDE } from "../../.."; +import { walkDir } from "../../walkDir"; +import { PythonPackageCrawler } from "./packageCrawlers/Python"; +import { TypeScriptPackageCrawler } from "./packageCrawlers/TsJs"; + +const PACKAGE_CRAWLERS = [TypeScriptPackageCrawler, PythonPackageCrawler]; + +export interface PackageCrawler { + language: string; + getPackageFiles(fileNames: string[]): string[]; + parsePackageFile(fileContent: string, filePath: string): PackageInfo[]; + getDocumentationLink(packageName: string): Promise<PackageDocsResult>; +} + +export type PackageInfo = { + name: string; + version: string; + foundInFilepath: string; +}; +export type PackageDocsResult = PackageInfo & + ({ error: string; link?: never } | { link: string; error?: never }); + +export async function getAllSuggestedDocs(ide: IDE) { + const workspaceDirs = await ide.getWorkspaceDirs(); + const results = await Promise.all( + workspaceDirs.map((dir) => { + return walkDir(dir, ide); + }), + ); + const allPaths = results.flat(); // TODO only get files, not dirs. Not critical for now + const allFiles = allPaths.map((path) => path.split("/").pop()!); + const packageFilesByLanguage: Record<string, string[]> = {}; + for (const Crawler of PACKAGE_CRAWLERS) { + const crawler = new Crawler(); + const packageFilePaths = crawler.getPackageFiles(allFiles); + packageFilesByLanguage[crawler.language] = packageFilePaths; + } + + const uniqueFilePaths = Array.from( + new Set(Object.values(packageFilesByLanguage).flat()), + ); + const fileContentsArray = await Promise.all( + uniqueFilePaths.map(async (path) => { + const contents = await ide.readFile(path); + return { path, contents }; + }), + ); + const fileContents = new Map( + fileContentsArray.map(({ path, contents }) => [path, contents]), + ); + + const packagesByLanguage: Record<string, PackageInfo[]> = {}; + PACKAGE_CRAWLERS.forEach((Crawler) => { + const crawler = new Crawler(); + const packageFiles = packageFilesByLanguage[crawler.language]; + packageFiles.forEach((file) => { + const contents = fileContents.get(file); + if (!contents) { + return; + } + const packages = crawler.parsePackageFile(contents, file); + if (!packagesByLanguage[crawler.language]) { + packagesByLanguage[crawler.language] = []; + } + packagesByLanguage[crawler.language].push(...packages); + }); + }); + console.log(packagesByLanguage); + // const allPackages = await Promise.all( + // PACKAGE_CRAWLERS.map(async (Crawler) => { + // const crawler = new Crawler(); + // const packageInfos = uniqueFileContents + // .filter(({ path }) => languageToFilePaths[crawler.language].includes(path)) + // .map(({ path, contents }) => crawler.parsePackageFile(contents, path)) + // .flat(); + // return packageInfos; + // }) + // ); + // for (const Crawler of PACKAGE_CRAWLERS) { + // const crawler = new Crawler(); + // const packages = crawler.parsePackageFile() + // languageToFilePaths[crawler.language] = packageFilePaths; + // } +} + +// I want to present the user with a list of dependencies and allow them to select which ones to index (embed) documentation for. +// In order to prevent duplicate file reads, the process will be like this: +// 1. take in a list of filepaths called `filepaths` +// 2. loop an array of PackageCrawler classes to build a map of `language` (string) to `packageFilePaths` (string[]) +// 3. Get unique filepaths from `packageFilePaths` and build a map ` of filepath to file contents using an existing `readFile` function, and skipping file reads of already in the map +// Finally, +// Add a `` method to the interface and classes that returns +// Then, assemble the classes in an array, and write a function getAllSuggestedDocs that returns a map of `language` to an ar diff --git a/core/indexing/docs/suggestions/packageCrawlers/Python.ts b/core/indexing/docs/suggestions/packageCrawlers/Python.ts new file mode 100644 index 0000000000..ac06997663 --- /dev/null +++ b/core/indexing/docs/suggestions/packageCrawlers/Python.ts @@ -0,0 +1,33 @@ +import { PackageCrawler, PackageDocsResult, PackageInfo } from ".."; + +export class PythonPackageCrawler implements PackageCrawler { + language = "python"; + + getPackageFiles(fileNames: string[]): string[] { + // For Python, we typically look for files like requirements.txt or Pipfile + return fileNames.filter( + (fileName) => fileName === "requirements.txt" || fileName === "Pipfile", + ); + } + + parsePackageFile(fileContent: string, filepath: string): PackageInfo[] { + // Assume the fileContent is a string from a requirements.txt formatted file + return fileContent + .split("\n") + .map((line) => { + const [name, version] = line.split("=="); + return { name, version, foundInFilepath: filepath }; + }) + .filter((pkg) => pkg.name && pkg.version); + } + + async getDocumentationLink(packageName: string): Promise<PackageDocsResult> { + // Fetch metadata from PyPI to find the documentation link + const response = await fetch(`https://pypi.org/pypi/${packageName}/json`); + if (!response.ok) { + throw new Error(`Could not fetch data for package ${packageName}`); + } + const data = await response.json(); + return data.info.project_urls?.Documentation; + } +} diff --git a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts new file mode 100644 index 0000000000..fb073e2a1d --- /dev/null +++ b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts @@ -0,0 +1,33 @@ +import { PackageCrawler, PackageDocsResult, PackageInfo } from ".."; + +export class TypeScriptPackageCrawler implements PackageCrawler { + language = "typescript"; + + getPackageFiles(fileNames: string[]): string[] { + // For TypeScript, we look for package.json file + return fileNames.filter((fileName) => fileName === "package.json"); + } + + parsePackageFile(fileContent: string, filepath: string): PackageInfo[] { + // Parse the package.json content + const jsonData = JSON.parse(fileContent) as Record<string, Object>; + const dependencies = Object.entries(jsonData.dependencies || {}).concat( + Object.entries(jsonData.devDependencies || {}), + ); + return dependencies.map(([name, version]) => ({ + name, + version, + foundInFilepath: filepath, + })); + } + + async getDocumentationLink(packageName: string): Promise<PackageDocsResult> { + // Fetch metadata from the NPM registry to find the documentation link + const response = await fetch(`https://registry.npmjs.org/${packageName}`); + if (!response.ok) { + throw new Error(`Could not fetch data for package ${packageName}`); + } + const data = await response.json(); + return data.homepage; + } +} diff --git a/core/protocol/core.ts b/core/protocol/core.ts index c757e5d347..069f9e297a 100644 --- a/core/protocol/core.ts +++ b/core/protocol/core.ts @@ -167,6 +167,7 @@ export type ToCoreFromIdeOrWebviewProtocol = { "indexing/abort": [{ type: string; id: string }, void]; "indexing/setPaused": [{ type: string; id: string; paused: boolean }, void]; "indexing/initStatuses": [undefined, void]; + "docs/getSuggestedDocs": [undefined, void]; addAutocompleteModel: [{ model: ModelDescription }, void]; diff --git a/core/protocol/passThrough.ts b/core/protocol/passThrough.ts index 40d537d41b..ebe2a86977 100644 --- a/core/protocol/passThrough.ts +++ b/core/protocol/passThrough.ts @@ -52,6 +52,7 @@ export const WEBVIEW_TO_CORE_PASS_THROUGH: (keyof ToCoreFromWebviewProtocol)[] = "indexing/reindex", "indexing/abort", "indexing/setPaused", + "docs/getSuggestedDocs", // "completeOnboarding", "addAutocompleteModel", diff --git a/extensions/vscode/package-lock.json b/extensions/vscode/package-lock.json index 9405035051..b30a951dfc 100644 --- a/extensions/vscode/package-lock.json +++ b/extensions/vscode/package-lock.json @@ -1,12 +1,12 @@ { "name": "continue", - "version": "0.9.232", + "version": "0.9.233", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "continue", - "version": "0.9.232", + "version": "0.9.233", "license": "Apache-2.0", "dependencies": { "@electron/rebuild": "^3.2.10", diff --git a/gui/src/hooks/useSetup.ts b/gui/src/hooks/useSetup.ts index 4bef9da9ce..f183798890 100644 --- a/gui/src/hooks/useSetup.ts +++ b/gui/src/hooks/useSetup.ts @@ -84,6 +84,7 @@ function useSetup(dispatch: Dispatch) { // ON LOAD useEffect(() => { + ideMessenger.post("docs/getSuggestedDocs", undefined); // Override persisted state dispatch(setInactive()); @@ -178,8 +179,6 @@ function useSetup(dispatch: Dispatch) { }, [defaultModelTitle], ); - - } export default useSetup; diff --git a/manual-testing-sandbox/package.json b/manual-testing-sandbox/package.json new file mode 100644 index 0000000000..ef77ac910c --- /dev/null +++ b/manual-testing-sandbox/package.json @@ -0,0 +1,102 @@ +{ + "name": "gui", + "private": true, + "type": "module", + "author": "Continue Dev, Inc", + "license": "Apache-2.0", + "scripts": { + "dev": "vite", + "tsc:check": "tsc -p ./ --noEmit", + "build": "tsc && vite build", + "preview": "vite preview", + "test": "vitest run", + "test:coverage": "vitest run --coverage", + "test:ui": "vitest --ui", + "test:watch": "vitest" + }, + "dependencies": { + "@headlessui/react": "^1.7.17", + "@heroicons/react": "^2.0.18", + "@monaco-editor/react": "^4.6.0", + "@reduxjs/toolkit": "^1.9.3", + "@tiptap/core": "^2.3.2", + "@tiptap/extension-document": "^2.3.2", + "@tiptap/extension-dropcursor": "^2.1.16", + "@tiptap/extension-history": "^2.3.2", + "@tiptap/extension-image": "^2.1.16", + "@tiptap/extension-mention": "^2.1.13", + "@tiptap/extension-paragraph": "^2.3.2", + "@tiptap/extension-placeholder": "^2.1.13", + "@tiptap/extension-text": "^2.3.2", + "@tiptap/pm": "^2.1.13", + "@tiptap/react": "^2.1.13", + "@tiptap/starter-kit": "^2.1.13", + "@tiptap/suggestion": "^2.1.13", + "@types/vscode-webview": "^1.57.1", + "core": "file:../core", + "dompurify": "^3.0.6", + "downshift": "^7.6.0", + "lodash": "^4.17.21", + "minisearch": "^7.0.2", + "onigasm": "^2.2.5", + "posthog-js": "^1.130.1", + "prismjs": "^1.29.0", + "react": "^18.2.0", + "react-dom": "^18.2.0", + "react-error-boundary": "^4.0.11", + "react-hook-form": "^7.47.0", + "react-intersection-observer": "^9.13.1", + "react-markdown": "^9.0.1", + "react-redux": "^8.0.5", + "react-remark": "^2.1.0", + "react-router-dom": "^6.14.2", + "react-switch": "^7.0.0", + "react-syntax-highlighter": "^15.5.0", + "react-tooltip": "^5.18.0", + "redux-persist": "^6.0.0", + "redux-persist-transform-filter": "^0.0.22", + "rehype-highlight": "^7.0.0", + "rehype-katex": "^7.0.0", + "rehype-wrap-all": "^1.1.0", + "remark-math": "^6.0.0", + "reselect": "^5.1.1", + "seti-file-icons": "^0.0.8", + "socket.io-client": "^4.7.2", + "styled-components": "^5.3.6", + "table": "^6.8.1", + "tippy.js": "^6.3.7", + "unist-util-visit": "^5.0.0", + "uuid": "^9.0.1", + "vscode-webview": "^1.0.1-beta.1" + }, + "devDependencies": { + "@swc/cli": "^0.3.14", + "@swc/core": "^1.7.26", + "@testing-library/dom": "^10.4.0", + "@testing-library/jest-dom": "^6.5.0", + "@testing-library/react": "^16.0.1", + "@testing-library/user-event": "^14.5.2", + "@types/lodash": "^4.17.6", + "@types/node": "^20.5.6", + "@types/node-fetch": "^2.6.4", + "@types/react": "^18.3.11", + "@types/react-dom": "^18.3.1", + "@types/react-router-dom": "^5.3.3", + "@types/react-syntax-highlighter": "^15.5.7", + "@types/styled-components": "^5.1.26", + "@vitejs/plugin-react-swc": "^3.7.0", + "@vitest/coverage-v8": "^2.1.3", + "@vitest/ui": "^2.1.3", + "autoprefixer": "^10.4.13", + "jsdom": "^25.0.1", + "postcss": "^8.4.21", + "tailwindcss": "^3.2.7", + "typescript": "^4.9.3", + "vite": "^4.1.0", + "vitest": "^2.1.3" + }, + "engine-strict": true, + "engines": { + "node": ">=20.11.0" + } +} diff --git a/manual-testing-sandbox/requirements.txt b/manual-testing-sandbox/requirements.txt new file mode 100644 index 0000000000..793d6a8275 --- /dev/null +++ b/manual-testing-sandbox/requirements.txt @@ -0,0 +1,10 @@ +flask==2.1.1 +requests==2.28.1 +numpy==1.23.4 +pandas==1.5.0 +scipy==1.9.3 +django==4.1.3 +matplotlib==3.6.2 +pytest==7.2.0 +sqlalchemy==1.4.41 +beautifulsoup4==4.11.1 \ No newline at end of file From 954bd5f563a278c04f40f0ec40bfbe27fc497afb Mon Sep 17 00:00:00 2001 From: Dallin Romney <dallinromney@gmail.com> Date: Thu, 21 Nov 2024 11:58:48 -0800 Subject: [PATCH 02/17] python and typescript crawlers --- core/index.d.ts | 29 +++++ core/indexing/docs/suggestions/index.ts | 115 +++++++++++------- .../suggestions/packageCrawlers/Python.ts | 39 ++++-- .../docs/suggestions/packageCrawlers/TsJs.ts | 40 ++++-- 4 files changed, 159 insertions(+), 64 deletions(-) diff --git a/core/index.d.ts b/core/index.d.ts index 0646edc541..de1adb8934 100644 --- a/core/index.d.ts +++ b/core/index.d.ts @@ -1190,3 +1190,32 @@ export interface BrowserSerializedContinueConfig { experimental?: ExperimentalConfig; analytics?: AnalyticsConfig; } + +// DOCS SUGGESTIONS AND PACKAGE INFO +export interface FilePathAndName { + path: string; + name: string; +} +export type ParsedPackageInfo = { + language: string; + name: string; + version: string; + packageFile: FilePathAndName; +}; + +export type PackageDetails = { + docsLink?: string; + title?: string; + description?: string; +}; + +export type PackageDetailsSuccess = PackageDetails & { + docsLink: string; +}; + +export type PackageDocsResult = { + packageInfo: ParsedPackageInfo; +} & ( + | { error: string; details?: never } + | { details: PackageDetailsSuccess; error?: never } +); diff --git a/core/indexing/docs/suggestions/index.ts b/core/indexing/docs/suggestions/index.ts index d3d68aca29..5699cd761c 100644 --- a/core/indexing/docs/suggestions/index.ts +++ b/core/indexing/docs/suggestions/index.ts @@ -1,12 +1,12 @@ -// write me an interface PackageCrawler that contains: -// 1. property `language` to store a given language like "python" or "typescript" -// 2. has a method `getPackageFiles` which takes a list of file names and decides which ones match package/dependency files (e.g. package.json for typescript, requirements.txt for python, etc) -// 3. has a method `parsePackageFile` which returns a list of package name and version from a relevant package file, in a standardized format like semver -// 4. has a method `getDocumentationLink` to check for documentation link for a given package (e.g. GET `https://registry.npmjs.org/<package>` and find docs field for typescript, documentation link in the package metadata for PyPi, etc.) -// Then, write typescript classes to implement this typescript interface for the languages "python" and "typescript" - -import { IDE } from "../../.."; +import { + FilePathAndName, + IDE, + PackageDetails, + PackageDocsResult, + ParsedPackageInfo, +} from "../../.."; import { walkDir } from "../../walkDir"; + import { PythonPackageCrawler } from "./packageCrawlers/Python"; import { TypeScriptPackageCrawler } from "./packageCrawlers/TsJs"; @@ -14,19 +14,14 @@ const PACKAGE_CRAWLERS = [TypeScriptPackageCrawler, PythonPackageCrawler]; export interface PackageCrawler { language: string; - getPackageFiles(fileNames: string[]): string[]; - parsePackageFile(fileContent: string, filePath: string): PackageInfo[]; - getDocumentationLink(packageName: string): Promise<PackageDocsResult>; + getPackageFiles(files: FilePathAndName[]): FilePathAndName[]; + parsePackageFile( + file: FilePathAndName, + contents: string, + ): ParsedPackageInfo[]; + getPackageDetails(packageInfo: ParsedPackageInfo): Promise<PackageDetails>; } -export type PackageInfo = { - name: string; - version: string; - foundInFilepath: string; -}; -export type PackageDocsResult = PackageInfo & - ({ error: string; link?: never } | { link: string; error?: never }); - export async function getAllSuggestedDocs(ide: IDE) { const workspaceDirs = await ide.getWorkspaceDirs(); const results = await Promise.all( @@ -35,16 +30,26 @@ export async function getAllSuggestedDocs(ide: IDE) { }), ); const allPaths = results.flat(); // TODO only get files, not dirs. Not critical for now - const allFiles = allPaths.map((path) => path.split("/").pop()!); - const packageFilesByLanguage: Record<string, string[]> = {}; + const allFiles = allPaths.map((path) => ({ + path, + name: path.split(/[\\/]/).pop()!, + })); + + // Build map of language -> package files + const packageFilesByLanguage: Record<string, FilePathAndName[]> = {}; for (const Crawler of PACKAGE_CRAWLERS) { const crawler = new Crawler(); const packageFilePaths = crawler.getPackageFiles(allFiles); packageFilesByLanguage[crawler.language] = packageFilePaths; } + // Get file contents for all unique package files const uniqueFilePaths = Array.from( - new Set(Object.values(packageFilesByLanguage).flat()), + new Set( + Object.values(packageFilesByLanguage).flatMap((files) => + files.map((file) => file.path), + ), + ), ); const fileContentsArray = await Promise.all( uniqueFilePaths.map(async (path) => { @@ -56,40 +61,68 @@ export async function getAllSuggestedDocs(ide: IDE) { fileContentsArray.map(({ path, contents }) => [path, contents]), ); - const packagesByLanguage: Record<string, PackageInfo[]> = {}; + // Parse package files and build map of language -> packages + const packagesByLanguage: Record<string, ParsedPackageInfo[]> = {}; PACKAGE_CRAWLERS.forEach((Crawler) => { const crawler = new Crawler(); const packageFiles = packageFilesByLanguage[crawler.language]; packageFiles.forEach((file) => { - const contents = fileContents.get(file); + const contents = fileContents.get(file.path); if (!contents) { return; } - const packages = crawler.parsePackageFile(contents, file); + const packages = crawler.parsePackageFile(file, contents); if (!packagesByLanguage[crawler.language]) { packagesByLanguage[crawler.language] = []; } packagesByLanguage[crawler.language].push(...packages); }); }); - console.log(packagesByLanguage); - // const allPackages = await Promise.all( - // PACKAGE_CRAWLERS.map(async (Crawler) => { - // const crawler = new Crawler(); - // const packageInfos = uniqueFileContents - // .filter(({ path }) => languageToFilePaths[crawler.language].includes(path)) - // .map(({ path, contents }) => crawler.parsePackageFile(contents, path)) - // .flat(); - // return packageInfos; - // }) - // ); - // for (const Crawler of PACKAGE_CRAWLERS) { - // const crawler = new Crawler(); - // const packages = crawler.parsePackageFile() - // languageToFilePaths[crawler.language] = packageFilePaths; - // } + + // Get documentation links for all packages + const docsByLanguage: Record<string, PackageDocsResult[]> = {}; + await Promise.all( + PACKAGE_CRAWLERS.map(async (Crawler) => { + const crawler = new Crawler(); + const packages = packagesByLanguage[crawler.language]; + docsByLanguage[crawler.language] = await Promise.all( + packages.map(async (packageInfo) => { + try { + const details = await crawler.getPackageDetails(packageInfo); + if (!details.docsLink) { + return { + packageInfo, + error: `No documentation link found for ${packageInfo.name}`, + }; + } + return { + packageInfo, + details: { + ...details, + docsLink: details.docsLink, + }, + }; + } catch (error) { + return { + packageInfo, + error: `Error getting package details for ${name}`, + }; + } + }), + ); + }), + ); + + return docsByLanguage; } +// write me an interface PackageCrawler that contains: +// 1. property `language` to store a given language like "python" or "typescript" +// 2. has a method `getPackageFiles` which takes a list of file names and decides which ones match package/dependency files (e.g. package.json for typescript, requirements.txt for python, etc) +// 3. has a method `parsePackageFile` which returns a list of package name and version from a relevant package file, in a standardized format like semver +// 4. has a method `getDocumentationLink` to check for documentation link for a given package (e.g. GET `https://registry.npmjs.org/<package>` and find docs field for typescript, documentation link in the package metadata for PyPi, etc.) +// Then, write typescript classes to implement this typescript interface for the languages "python" and "typescript" + // I want to present the user with a list of dependencies and allow them to select which ones to index (embed) documentation for. // In order to prevent duplicate file reads, the process will be like this: // 1. take in a list of filepaths called `filepaths` diff --git a/core/indexing/docs/suggestions/packageCrawlers/Python.ts b/core/indexing/docs/suggestions/packageCrawlers/Python.ts index ac06997663..233e444285 100644 --- a/core/indexing/docs/suggestions/packageCrawlers/Python.ts +++ b/core/indexing/docs/suggestions/packageCrawlers/Python.ts @@ -1,33 +1,50 @@ -import { PackageCrawler, PackageDocsResult, PackageInfo } from ".."; +import { PackageCrawler } from ".."; +import { + FilePathAndName, + PackageDetails, + ParsedPackageInfo, +} from "../../../.."; export class PythonPackageCrawler implements PackageCrawler { language = "python"; - getPackageFiles(fileNames: string[]): string[] { + getPackageFiles(files: FilePathAndName[]): FilePathAndName[] { // For Python, we typically look for files like requirements.txt or Pipfile - return fileNames.filter( - (fileName) => fileName === "requirements.txt" || fileName === "Pipfile", + return files.filter( + (file) => file.name === "requirements.txt" || file.name === "Pipfile", ); } - parsePackageFile(fileContent: string, filepath: string): PackageInfo[] { + parsePackageFile( + file: FilePathAndName, + contents: string, + ): ParsedPackageInfo[] { // Assume the fileContent is a string from a requirements.txt formatted file - return fileContent + return contents .split("\n") .map((line) => { const [name, version] = line.split("=="); - return { name, version, foundInFilepath: filepath }; + return { name, version, packageFile: file, language: this.language }; }) .filter((pkg) => pkg.name && pkg.version); } - async getDocumentationLink(packageName: string): Promise<PackageDocsResult> { + async getPackageDetails( + packageInfo: ParsedPackageInfo, + ): Promise<PackageDetails> { // Fetch metadata from PyPI to find the documentation link - const response = await fetch(`https://pypi.org/pypi/${packageName}/json`); + + const response = await fetch( + `https://pypi.org/pypi/${packageInfo.name}/json`, + ); if (!response.ok) { - throw new Error(`Could not fetch data for package ${packageName}`); + throw new Error(`Could not fetch data for package ${packageInfo.name}`); } const data = await response.json(); - return data.info.project_urls?.Documentation; + return { + docsLink: data.info.project_urls?.Documentation as string | undefined, + // title: data.info.name, + // description: data.info.summary as string | undefined, + }; } } diff --git a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts index fb073e2a1d..107d9ac2a1 100644 --- a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts +++ b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts @@ -1,33 +1,49 @@ -import { PackageCrawler, PackageDocsResult, PackageInfo } from ".."; +import { PackageCrawler } from ".."; +import { + FilePathAndName, + PackageDetails, + ParsedPackageInfo, +} from "../../../.."; export class TypeScriptPackageCrawler implements PackageCrawler { - language = "typescript"; + language = "js-ts"; - getPackageFiles(fileNames: string[]): string[] { - // For TypeScript, we look for package.json file - return fileNames.filter((fileName) => fileName === "package.json"); + getPackageFiles(files: FilePathAndName[]): FilePathAndName[] { + // For Javascript/TypeScript, we look for package.json file + return files.filter((file) => file.name === "package.json"); } - parsePackageFile(fileContent: string, filepath: string): PackageInfo[] { + parsePackageFile( + file: FilePathAndName, + contents: string, + ): ParsedPackageInfo[] { // Parse the package.json content - const jsonData = JSON.parse(fileContent) as Record<string, Object>; + const jsonData = JSON.parse(contents) as Record<string, Object>; const dependencies = Object.entries(jsonData.dependencies || {}).concat( Object.entries(jsonData.devDependencies || {}), ); return dependencies.map(([name, version]) => ({ name, version, - foundInFilepath: filepath, + packageFile: file, + language: this.language, })); } - async getDocumentationLink(packageName: string): Promise<PackageDocsResult> { + async getPackageDetails( + packageInfo: ParsedPackageInfo, + ): Promise<PackageDetails> { + const { name } = packageInfo; // Fetch metadata from the NPM registry to find the documentation link - const response = await fetch(`https://registry.npmjs.org/${packageName}`); + const response = await fetch(`https://registry.npmjs.org/${name}`); if (!response.ok) { - throw new Error(`Could not fetch data for package ${packageName}`); + throw new Error(`Could not fetch data for package ${name}`); } const data = await response.json(); - return data.homepage; + return { + docsLink: data.homepage as string | undefined, + // title: data.name, + // description: data.description as string | undefined, + }; } } From abd46dfb4ff1aea84e94922fdda5d6fce23c6edf Mon Sep 17 00:00:00 2001 From: Dallin Romney <dallinromney@gmail.com> Date: Thu, 21 Nov 2024 12:52:25 -0800 Subject: [PATCH 03/17] docs suggestions: packages in GUI --- core/core.ts | 3 +- core/index.d.ts | 3 ++ core/indexing/docs/suggestions/index.ts | 4 +-- core/protocol/passThrough.ts | 1 + core/protocol/webview.ts | 2 ++ gui/src/components/dialogs/AddDocsDialog.tsx | 37 ++++++++++++++++++-- gui/src/hooks/useSetup.ts | 6 ++++ gui/src/redux/slices/stateSlice.ts | 11 ++++++ 8 files changed, 61 insertions(+), 6 deletions(-) diff --git a/core/core.ts b/core/core.ts index d080ec34b0..d81df8a379 100644 --- a/core/core.ts +++ b/core/core.ts @@ -739,7 +739,8 @@ export class Core { return this.docsService.initStatuses(); }); on("docs/getSuggestedDocs", async (msg) => { - getAllSuggestedDocs(this.ide); + const suggestedDocs = await getAllSuggestedDocs(this.ide); + this.messenger.send("docs/suggestions", suggestedDocs); }); // diff --git a/core/index.d.ts b/core/index.d.ts index de1adb8934..d6b2c1970c 100644 --- a/core/index.d.ts +++ b/core/index.d.ts @@ -1219,3 +1219,6 @@ export type PackageDocsResult = { | { error: string; details?: never } | { details: PackageDetailsSuccess; error?: never } ); + +// language -> package -> package info +export type DocsSuggestions = Record<string, PackageDocsResult[]>; diff --git a/core/indexing/docs/suggestions/index.ts b/core/indexing/docs/suggestions/index.ts index 5699cd761c..3ee4fb6426 100644 --- a/core/indexing/docs/suggestions/index.ts +++ b/core/indexing/docs/suggestions/index.ts @@ -1,4 +1,5 @@ import { + DocsSuggestions, FilePathAndName, IDE, PackageDetails, @@ -80,7 +81,7 @@ export async function getAllSuggestedDocs(ide: IDE) { }); // Get documentation links for all packages - const docsByLanguage: Record<string, PackageDocsResult[]> = {}; + const docsByLanguage: DocsSuggestions = {}; await Promise.all( PACKAGE_CRAWLERS.map(async (Crawler) => { const crawler = new Crawler(); @@ -112,7 +113,6 @@ export async function getAllSuggestedDocs(ide: IDE) { ); }), ); - return docsByLanguage; } diff --git a/core/protocol/passThrough.ts b/core/protocol/passThrough.ts index ebe2a86977..2bd8de2c18 100644 --- a/core/protocol/passThrough.ts +++ b/core/protocol/passThrough.ts @@ -75,4 +75,5 @@ export const CORE_TO_WEBVIEW_PASS_THROUGH: (keyof ToWebviewFromCoreProtocol)[] = "getWebviewHistoryLength", "signInToControlPlane", "openDialogMessage", + "docs/suggestions", ]; diff --git a/core/protocol/webview.ts b/core/protocol/webview.ts index 0dcdb047f6..bf90497c8a 100644 --- a/core/protocol/webview.ts +++ b/core/protocol/webview.ts @@ -2,6 +2,7 @@ import { ConfigValidationError } from "../config/validation.js"; import type { ContextItemWithId, + DocsSuggestions, IndexingProgressUpdate, IndexingStatus, } from "../index.js"; @@ -25,4 +26,5 @@ export type ToWebviewFromIdeOrCoreProtocol = { getWebviewHistoryLength: [undefined, number]; signInToControlPlane: [undefined, void]; openDialogMessage: ["account", void]; + "docs/suggestions": [DocsSuggestions, void]; }; diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx index 0981948869..c21aca5a6c 100644 --- a/gui/src/components/dialogs/AddDocsDialog.tsx +++ b/gui/src/components/dialogs/AddDocsDialog.tsx @@ -5,7 +5,7 @@ import { } from "@heroicons/react/24/outline"; import { SiteIndexingConfig } from "core"; import { usePostHog } from "posthog-js/react"; -import { useContext, useLayoutEffect, useRef, useState } from "react"; +import { useContext, useLayoutEffect, useMemo, useRef, useState } from "react"; import { useDispatch, useSelector } from "react-redux"; import { Button, HelperText, Input, lightGray, SecondaryButton } from ".."; import { IdeMessengerContext } from "../../context/IdeMessenger"; @@ -15,6 +15,7 @@ import { } from "../../redux/slices/uiStateSlice"; import { RootState } from "../../redux/store"; import IndexingStatusViewer from "../indexing/IndexingStatus"; +import { C } from "core/autocomplete/constants/AutocompleteLanguageInfo"; function AddDocsDialog() { const posthog = usePostHog(); @@ -34,6 +35,21 @@ function AddDocsDialog() { (store: RootState) => store.state.indexing.statuses, ); + const docsSuggestions = useSelector( + (store: RootState) => store.state.docsSuggestions, + ); + + const docsByLanguage = useMemo(() => { + console.log(docsSuggestions); + const languages = Object.keys(docsSuggestions); + return languages.map((language) => { + return { + language, + packages: docsSuggestions[language], + }; + }); + }, [docsSuggestions]); + const isFormValid = startUrl && title; useLayoutEffect(() => { @@ -118,8 +134,23 @@ function AddDocsDialog() { return ( <div className="p-4"> <div className="mb-8"> - <h1>Add a documentation site</h1> - + <h1>Add documentation</h1> + {docsByLanguage.map(({ language, packages }) => { + return ( + <div key={language}> + <h1>{language}</h1> + <div> + {packages.map((pkg) => { + return ( + <div> + <p>{pkg.packageInfo.name}</p> + </div> + ); + })} + </div> + </div> + ); + })} <p> Continue pre-indexes many common documentation sites, but if there's one you don't see in the dropdown, enter the URL here. diff --git a/gui/src/hooks/useSetup.ts b/gui/src/hooks/useSetup.ts index f183798890..ecc7a7ac92 100644 --- a/gui/src/hooks/useSetup.ts +++ b/gui/src/hooks/useSetup.ts @@ -11,6 +11,7 @@ import { setInactive, setSelectedProfileId, setTTSActive, + updateDocsSuggestions, updateIndexingStatus, } from "../redux/slices/stateSlice"; import { RootState } from "../redux/store"; @@ -85,6 +86,7 @@ function useSetup(dispatch: Dispatch) { // ON LOAD useEffect(() => { ideMessenger.post("docs/getSuggestedDocs", undefined); + // Override persisted state dispatch(setInactive()); @@ -116,6 +118,10 @@ function useSetup(dispatch: Dispatch) { } }, []); + useWebviewListener("docs/suggestions", async (data) => { + dispatch(updateDocsSuggestions(data)); + }); + const { streamResponse } = useChatHandler(dispatch, ideMessenger); const defaultModelTitle = useSelector( diff --git a/gui/src/redux/slices/stateSlice.ts b/gui/src/redux/slices/stateSlice.ts index 770e9ce698..e965455c4f 100644 --- a/gui/src/redux/slices/stateSlice.ts +++ b/gui/src/redux/slices/stateSlice.ts @@ -5,6 +5,7 @@ import { ChatMessage, Checkpoint, ContextItemWithId, + DocsSuggestions, FileSymbolMap, IndexingStatus, PersistedSessionInfo, @@ -47,6 +48,7 @@ type State = { }; streamAborter: AbortController; isMultifileEdit: boolean; + docsSuggestions: DocsSuggestions; }; const initialState: State = { @@ -89,6 +91,7 @@ const initialState: State = { }, }, streamAborter: new AbortController(), + docsSuggestions: {}, }; export const stateSlice = createSlice({ @@ -447,6 +450,13 @@ export const stateSlice = createSlice({ [payload.type]: payload.hidden, }; }, + updateDocsSuggestions: ( + state, + { payload }: PayloadAction<DocsSuggestions>, + ) => { + console.log("FROM REDUX", payload); + state.docsSuggestions = payload; + }, }, }); @@ -480,6 +490,7 @@ export const { updateIndexingStatus, setIndexingChatPeekHidden, abortStream, + updateDocsSuggestions, } = stateSlice.actions; export default stateSlice.reducer; From 53f3638b8f0ec703579f3136f7293d4d88ea8a28 Mon Sep 17 00:00:00 2001 From: Dallin Romney <dallinromney@gmail.com> Date: Thu, 21 Nov 2024 16:37:28 -0800 Subject: [PATCH 04/17] suggested docs continued --- core/index.d.ts | 11 +++++-- core/indexing/docs/suggestions/index.ts | 30 ++++++++++++----- .../suggestions/packageCrawlers/Python.ts | 29 +++++++++++++---- .../docs/suggestions/packageCrawlers/TsJs.ts | 24 +++++++++++--- gui/src/components/dialogs/AddDocsDialog.tsx | 32 ++++--------------- gui/src/components/dialogs/SuggestedDocs.tsx | 18 +++++++++++ gui/src/redux/slices/stateSlice.ts | 9 +++--- 7 files changed, 100 insertions(+), 53 deletions(-) create mode 100644 gui/src/components/dialogs/SuggestedDocs.tsx diff --git a/core/index.d.ts b/core/index.d.ts index d6b2c1970c..328462e70e 100644 --- a/core/index.d.ts +++ b/core/index.d.ts @@ -1196,6 +1196,12 @@ export interface FilePathAndName { path: string; name: string; } + +export interface PackageFilePathAndName extends FilePathAndName { + language: string; // e.g. javascript + registry: string; // e.g. npm +} + export type ParsedPackageInfo = { language: string; name: string; @@ -1207,6 +1213,8 @@ export type PackageDetails = { docsLink?: string; title?: string; description?: string; + repo?: string; + license?: string; }; export type PackageDetailsSuccess = PackageDetails & { @@ -1219,6 +1227,3 @@ export type PackageDocsResult = { | { error: string; details?: never } | { details: PackageDetailsSuccess; error?: never } ); - -// language -> package -> package info -export type DocsSuggestions = Record<string, PackageDocsResult[]>; diff --git a/core/indexing/docs/suggestions/index.ts b/core/indexing/docs/suggestions/index.ts index 3ee4fb6426..0f86f53902 100644 --- a/core/indexing/docs/suggestions/index.ts +++ b/core/indexing/docs/suggestions/index.ts @@ -1,9 +1,9 @@ import { - DocsSuggestions, + PackageDocsResult, FilePathAndName, + PackageFilePathAndName, IDE, PackageDetails, - PackageDocsResult, ParsedPackageInfo, } from "../../.."; import { walkDir } from "../../walkDir"; @@ -15,9 +15,9 @@ const PACKAGE_CRAWLERS = [TypeScriptPackageCrawler, PythonPackageCrawler]; export interface PackageCrawler { language: string; - getPackageFiles(files: FilePathAndName[]): FilePathAndName[]; + getPackageFiles(files: FilePathAndName[]): PackageFilePathAndName[]; parsePackageFile( - file: FilePathAndName, + file: PackageFilePathAndName, contents: string, ): ParsedPackageInfo[]; getPackageDetails(packageInfo: ParsedPackageInfo): Promise<PackageDetails>; @@ -37,7 +37,7 @@ export async function getAllSuggestedDocs(ide: IDE) { })); // Build map of language -> package files - const packageFilesByLanguage: Record<string, FilePathAndName[]> = {}; + const packageFilesByLanguage: Record<string, PackageFilePathAndName[]> = {}; for (const Crawler of PACKAGE_CRAWLERS) { const crawler = new Crawler(); const packageFilePaths = crawler.getPackageFiles(allFiles); @@ -80,13 +80,26 @@ export async function getAllSuggestedDocs(ide: IDE) { }); }); + // Deduplicate packages per language + // TODO - this is where you would allow docs for different versions + // by e.g. using "name-version" as the map key instead of just name + // For now have not allowed + const languages = Object.keys(packagesByLanguage); + languages.forEach((language) => { + const packages = packagesByLanguage[language]; + const uniquePackages = Array.from( + new Map(packages.map((pkg) => [pkg.name, pkg])).values(), + ); + packagesByLanguage[language] = uniquePackages; + }); + // Get documentation links for all packages - const docsByLanguage: DocsSuggestions = {}; + const allDocsResults: PackageDocsResult[] = []; await Promise.all( PACKAGE_CRAWLERS.map(async (Crawler) => { const crawler = new Crawler(); const packages = packagesByLanguage[crawler.language]; - docsByLanguage[crawler.language] = await Promise.all( + const docsByLanguage = await Promise.all( packages.map(async (packageInfo) => { try { const details = await crawler.getPackageDetails(packageInfo); @@ -111,9 +124,10 @@ export async function getAllSuggestedDocs(ide: IDE) { } }), ); + allDocsResults.push(...docsByLanguage); }), ); - return docsByLanguage; + return allDocsResults; } // write me an interface PackageCrawler that contains: diff --git a/core/indexing/docs/suggestions/packageCrawlers/Python.ts b/core/indexing/docs/suggestions/packageCrawlers/Python.ts index 233e444285..28b893b66b 100644 --- a/core/indexing/docs/suggestions/packageCrawlers/Python.ts +++ b/core/indexing/docs/suggestions/packageCrawlers/Python.ts @@ -2,17 +2,24 @@ import { PackageCrawler } from ".."; import { FilePathAndName, PackageDetails, + PackageFilePathAndName, ParsedPackageInfo, } from "../../../.."; export class PythonPackageCrawler implements PackageCrawler { language = "python"; - getPackageFiles(files: FilePathAndName[]): FilePathAndName[] { + getPackageFiles(files: FilePathAndName[]): PackageFilePathAndName[] { // For Python, we typically look for files like requirements.txt or Pipfile - return files.filter( - (file) => file.name === "requirements.txt" || file.name === "Pipfile", - ); + return files + .filter( + (file) => file.name === "requirements.txt" || file.name === "Pipfile", + ) + .map((file) => ({ + ...file, + language: this.language, + registry: "pypi", + })); } parsePackageFile( @@ -41,10 +48,18 @@ export class PythonPackageCrawler implements PackageCrawler { throw new Error(`Could not fetch data for package ${packageInfo.name}`); } const data = await response.json(); + const homePage = data?.info?.home_page as string | undefined; + return { - docsLink: data.info.project_urls?.Documentation as string | undefined, - // title: data.info.name, - // description: data.info.summary as string | undefined, + docsLink: + (data?.info?.project_urls?.Documentation as string | undefined) ?? + homePage, + title: data?.info?.name as string | undefined, + description: data?.info?.summary as string | undefined, + repo: + (data?.info?.project_urls?.Repository as string | undefined) ?? + homePage, + license: data?.info?.license as string | undefined, }; } } diff --git a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts index 107d9ac2a1..c16b8872e8 100644 --- a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts +++ b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts @@ -2,15 +2,22 @@ import { PackageCrawler } from ".."; import { FilePathAndName, PackageDetails, + PackageFilePathAndName, ParsedPackageInfo, } from "../../../.."; export class TypeScriptPackageCrawler implements PackageCrawler { language = "js-ts"; - getPackageFiles(files: FilePathAndName[]): FilePathAndName[] { + getPackageFiles(files: FilePathAndName[]): PackageFilePathAndName[] { // For Javascript/TypeScript, we look for package.json file - return files.filter((file) => file.name === "package.json"); + return files + .filter((file) => file.name === "package.json") + .map((file) => ({ + ...file, + language: this.language, + registry: "npm", + })); } parsePackageFile( @@ -22,7 +29,10 @@ export class TypeScriptPackageCrawler implements PackageCrawler { const dependencies = Object.entries(jsonData.dependencies || {}).concat( Object.entries(jsonData.devDependencies || {}), ); - return dependencies.map(([name, version]) => ({ + const filtered = dependencies.filter( + ([name, version]) => !name.startsWith("@types/"), + ); + return filtered.map(([name, version]) => ({ name, version, packageFile: file, @@ -42,8 +52,12 @@ export class TypeScriptPackageCrawler implements PackageCrawler { const data = await response.json(); return { docsLink: data.homepage as string | undefined, - // title: data.name, - // description: data.description as string | undefined, + title: name, // package.json doesn't have specific title field + description: data.description as string | undefined, + repo: Array.isArray(data.repository) + ? (data.respository[0]?.url as string | undefined) + : undefined, + license: data.license as string | undefined, }; } } diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx index c21aca5a6c..7dcc478fc6 100644 --- a/gui/src/components/dialogs/AddDocsDialog.tsx +++ b/gui/src/components/dialogs/AddDocsDialog.tsx @@ -15,7 +15,6 @@ import { } from "../../redux/slices/uiStateSlice"; import { RootState } from "../../redux/store"; import IndexingStatusViewer from "../indexing/IndexingStatus"; -import { C } from "core/autocomplete/constants/AutocompleteLanguageInfo"; function AddDocsDialog() { const posthog = usePostHog(); @@ -39,17 +38,6 @@ function AddDocsDialog() { (store: RootState) => store.state.docsSuggestions, ); - const docsByLanguage = useMemo(() => { - console.log(docsSuggestions); - const languages = Object.keys(docsSuggestions); - return languages.map((language) => { - return { - language, - packages: docsSuggestions[language], - }; - }); - }, [docsSuggestions]); - const isFormValid = startUrl && title; useLayoutEffect(() => { @@ -135,22 +123,16 @@ function AddDocsDialog() { <div className="p-4"> <div className="mb-8"> <h1>Add documentation</h1> - {docsByLanguage.map(({ language, packages }) => { + {/* {docsSuggestions.map((docsResult) => { + const { error, details } = docsResult; + const { language, name, version } = docsResult.packageInfo; return ( - <div key={language}> - <h1>{language}</h1> - <div> - {packages.map((pkg) => { - return ( - <div> - <p>{pkg.packageInfo.name}</p> - </div> - ); - })} - </div> + <div key={`${language}-${name}-${version}`}> + <p className="m-0 p-0">{`${name}`}</p> + <p className="m-0 p-0">{error ? "error" : details.docsLink}</p> </div> ); - })} + })} */} <p> Continue pre-indexes many common documentation sites, but if there's one you don't see in the dropdown, enter the URL here. diff --git a/gui/src/components/dialogs/SuggestedDocs.tsx b/gui/src/components/dialogs/SuggestedDocs.tsx new file mode 100644 index 0000000000..1540ea92f9 --- /dev/null +++ b/gui/src/components/dialogs/SuggestedDocs.tsx @@ -0,0 +1,18 @@ +import { PackageDocsResult } from "core"; + +interface SuggestedDocProps { + docResult: PackageDocsResult; +} +const SuggestedDoc = ({ docResult }: SuggestedDocProps) => { + return <div>SuggestedDoc</div>; +}; + +interface SuggestedDocsListProps { + docs: PackageDocsResult[]; + // on +} +const SuggestedDocsList = ({ docs }: SuggestedDocsListProps) => { + return <div></div>; +}; + +export default SuggestedDocsList; diff --git a/gui/src/redux/slices/stateSlice.ts b/gui/src/redux/slices/stateSlice.ts index e965455c4f..2feaf80651 100644 --- a/gui/src/redux/slices/stateSlice.ts +++ b/gui/src/redux/slices/stateSlice.ts @@ -5,7 +5,7 @@ import { ChatMessage, Checkpoint, ContextItemWithId, - DocsSuggestions, + PackageDocsResult, FileSymbolMap, IndexingStatus, PersistedSessionInfo, @@ -48,7 +48,7 @@ type State = { }; streamAborter: AbortController; isMultifileEdit: boolean; - docsSuggestions: DocsSuggestions; + docsSuggestions: PackageDocsResult[]; }; const initialState: State = { @@ -91,7 +91,7 @@ const initialState: State = { }, }, streamAborter: new AbortController(), - docsSuggestions: {}, + docsSuggestions: [], }; export const stateSlice = createSlice({ @@ -452,9 +452,8 @@ export const stateSlice = createSlice({ }, updateDocsSuggestions: ( state, - { payload }: PayloadAction<DocsSuggestions>, + { payload }: PayloadAction<PackageDocsResult[]>, ) => { - console.log("FROM REDUX", payload); state.docsSuggestions = payload; }, }, From 48806f3f0831de8cf67546ef77467548187641cc Mon Sep 17 00:00:00 2001 From: Dallin Romney <dallinromney@gmail.com> Date: Thu, 21 Nov 2024 18:07:37 -0800 Subject: [PATCH 05/17] docs suggestions p4 --- core/core.ts | 8 +- core/index.d.ts | 7 +- core/indexing/docs/suggestions/index.ts | 36 ++-- .../suggestions/packageCrawlers/Python.ts | 9 +- .../docs/suggestions/packageCrawlers/TsJs.ts | 27 ++- gui/src/components/dialogs/AddDocsDialog.tsx | 186 ++++++++++++------ gui/src/components/index.ts | 10 + .../components/indexing/IndexingStatuses.tsx | 27 ++- gui/tailwind.config.cjs | 11 +- 9 files changed, 199 insertions(+), 122 deletions(-) diff --git a/core/core.ts b/core/core.ts index d81df8a379..b2be3478c3 100644 --- a/core/core.ts +++ b/core/core.ts @@ -19,6 +19,7 @@ import { ControlPlaneClient } from "./control-plane/client"; import { streamDiffLines } from "./edit/streamDiffLines"; import { CodebaseIndexer, PauseToken } from "./indexing/CodebaseIndexer"; import DocsService from "./indexing/docs/DocsService"; +import { getAllSuggestedDocs } from "./indexing/docs/suggestions"; import { defaultIgnoreFile } from "./indexing/ignore.js"; import Ollama from "./llm/llms/Ollama"; import { createNewPromptFileV2 } from "./promptFiles/v2/createNewPromptFile"; @@ -28,11 +29,7 @@ import { DevDataSqliteDb } from "./util/devdataSqlite"; import { fetchwithRequestOptions } from "./util/fetchWithOptions"; import { GlobalContext } from "./util/GlobalContext"; import historyManager from "./util/history"; -import { - editConfigJson, - getConfigJsonPath, - setupInitialDotContinueDirectory, -} from "./util/paths"; +import { editConfigJson, setupInitialDotContinueDirectory } from "./util/paths"; import { Telemetry } from "./util/posthog"; import { getSymbolsForManyFiles } from "./util/treeSitter"; import { TTS } from "./util/tts"; @@ -40,7 +37,6 @@ import { TTS } from "./util/tts"; import type { ContextItemId, IDE, IndexingProgressUpdate } from "."; import type { FromCoreProtocol, ToCoreProtocol } from "./protocol"; import type { IMessenger, Message } from "./util/messenger"; -import { getAllSuggestedDocs } from "./indexing/docs/suggestions"; export class Core { // implements IMessenger<ToCoreProtocol, FromCoreProtocol> diff --git a/core/index.d.ts b/core/index.d.ts index 328462e70e..5541584ebd 100644 --- a/core/index.d.ts +++ b/core/index.d.ts @@ -1198,15 +1198,14 @@ export interface FilePathAndName { } export interface PackageFilePathAndName extends FilePathAndName { - language: string; // e.g. javascript - registry: string; // e.g. npm + packageRegistry: string; // e.g. npm, pypi } export type ParsedPackageInfo = { - language: string; name: string; + packageFile: PackageFilePathAndName; + language: string; version: string; - packageFile: FilePathAndName; }; export type PackageDetails = { diff --git a/core/indexing/docs/suggestions/index.ts b/core/indexing/docs/suggestions/index.ts index 0f86f53902..4e0c2104ac 100644 --- a/core/indexing/docs/suggestions/index.ts +++ b/core/indexing/docs/suggestions/index.ts @@ -9,12 +9,12 @@ import { import { walkDir } from "../../walkDir"; import { PythonPackageCrawler } from "./packageCrawlers/Python"; -import { TypeScriptPackageCrawler } from "./packageCrawlers/TsJs"; +import { NodePackageCrawler } from "./packageCrawlers/TsJs"; -const PACKAGE_CRAWLERS = [TypeScriptPackageCrawler, PythonPackageCrawler]; +const PACKAGE_CRAWLERS = [NodePackageCrawler, PythonPackageCrawler]; export interface PackageCrawler { - language: string; + packageRegistry: string; getPackageFiles(files: FilePathAndName[]): PackageFilePathAndName[]; parsePackageFile( file: PackageFilePathAndName, @@ -37,17 +37,17 @@ export async function getAllSuggestedDocs(ide: IDE) { })); // Build map of language -> package files - const packageFilesByLanguage: Record<string, PackageFilePathAndName[]> = {}; + const packageFilesByRegistry: Record<string, PackageFilePathAndName[]> = {}; for (const Crawler of PACKAGE_CRAWLERS) { const crawler = new Crawler(); const packageFilePaths = crawler.getPackageFiles(allFiles); - packageFilesByLanguage[crawler.language] = packageFilePaths; + packageFilesByRegistry[crawler.packageRegistry] = packageFilePaths; } // Get file contents for all unique package files const uniqueFilePaths = Array.from( new Set( - Object.values(packageFilesByLanguage).flatMap((files) => + Object.values(packageFilesByRegistry).flatMap((files) => files.map((file) => file.path), ), ), @@ -63,20 +63,20 @@ export async function getAllSuggestedDocs(ide: IDE) { ); // Parse package files and build map of language -> packages - const packagesByLanguage: Record<string, ParsedPackageInfo[]> = {}; + const packagesByRegistry: Record<string, ParsedPackageInfo[]> = {}; PACKAGE_CRAWLERS.forEach((Crawler) => { const crawler = new Crawler(); - const packageFiles = packageFilesByLanguage[crawler.language]; + const packageFiles = packageFilesByRegistry[crawler.packageRegistry]; packageFiles.forEach((file) => { const contents = fileContents.get(file.path); if (!contents) { return; } const packages = crawler.parsePackageFile(file, contents); - if (!packagesByLanguage[crawler.language]) { - packagesByLanguage[crawler.language] = []; + if (!packagesByRegistry[crawler.packageRegistry]) { + packagesByRegistry[crawler.packageRegistry] = []; } - packagesByLanguage[crawler.language].push(...packages); + packagesByRegistry[crawler.packageRegistry].push(...packages); }); }); @@ -84,13 +84,13 @@ export async function getAllSuggestedDocs(ide: IDE) { // TODO - this is where you would allow docs for different versions // by e.g. using "name-version" as the map key instead of just name // For now have not allowed - const languages = Object.keys(packagesByLanguage); + const languages = Object.keys(packagesByRegistry); languages.forEach((language) => { - const packages = packagesByLanguage[language]; + const packages = packagesByRegistry[language]; const uniquePackages = Array.from( new Map(packages.map((pkg) => [pkg.name, pkg])).values(), ); - packagesByLanguage[language] = uniquePackages; + packagesByRegistry[language] = uniquePackages; }); // Get documentation links for all packages @@ -98,8 +98,8 @@ export async function getAllSuggestedDocs(ide: IDE) { await Promise.all( PACKAGE_CRAWLERS.map(async (Crawler) => { const crawler = new Crawler(); - const packages = packagesByLanguage[crawler.language]; - const docsByLanguage = await Promise.all( + const packages = packagesByRegistry[crawler.packageRegistry]; + const docsByRegistry = await Promise.all( packages.map(async (packageInfo) => { try { const details = await crawler.getPackageDetails(packageInfo); @@ -119,12 +119,12 @@ export async function getAllSuggestedDocs(ide: IDE) { } catch (error) { return { packageInfo, - error: `Error getting package details for ${name}`, + error: `Error getting package details for ${packageInfo.name}`, }; } }), ); - allDocsResults.push(...docsByLanguage); + allDocsResults.push(...docsByRegistry); }), ); return allDocsResults; diff --git a/core/indexing/docs/suggestions/packageCrawlers/Python.ts b/core/indexing/docs/suggestions/packageCrawlers/Python.ts index 28b893b66b..d00045f976 100644 --- a/core/indexing/docs/suggestions/packageCrawlers/Python.ts +++ b/core/indexing/docs/suggestions/packageCrawlers/Python.ts @@ -7,7 +7,7 @@ import { } from "../../../.."; export class PythonPackageCrawler implements PackageCrawler { - language = "python"; + packageRegistry = "pypi"; getPackageFiles(files: FilePathAndName[]): PackageFilePathAndName[] { // For Python, we typically look for files like requirements.txt or Pipfile @@ -17,13 +17,12 @@ export class PythonPackageCrawler implements PackageCrawler { ) .map((file) => ({ ...file, - language: this.language, - registry: "pypi", + packageRegistry: "pypi", })); } parsePackageFile( - file: FilePathAndName, + file: PackageFilePathAndName, contents: string, ): ParsedPackageInfo[] { // Assume the fileContent is a string from a requirements.txt formatted file @@ -31,7 +30,7 @@ export class PythonPackageCrawler implements PackageCrawler { .split("\n") .map((line) => { const [name, version] = line.split("=="); - return { name, version, packageFile: file, language: this.language }; + return { name, version, packageFile: file, language: "python" }; }) .filter((pkg) => pkg.name && pkg.version); } diff --git a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts index c16b8872e8..42c5007480 100644 --- a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts +++ b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts @@ -6,8 +6,8 @@ import { ParsedPackageInfo, } from "../../../.."; -export class TypeScriptPackageCrawler implements PackageCrawler { - language = "js-ts"; +export class NodePackageCrawler implements PackageCrawler { + packageRegistry = "npm"; getPackageFiles(files: FilePathAndName[]): PackageFilePathAndName[] { // For Javascript/TypeScript, we look for package.json file @@ -15,13 +15,12 @@ export class TypeScriptPackageCrawler implements PackageCrawler { .filter((file) => file.name === "package.json") .map((file) => ({ ...file, - language: this.language, - registry: "npm", + packageRegistry: this.packageRegistry, })); } parsePackageFile( - file: FilePathAndName, + file: PackageFilePathAndName, contents: string, ): ParsedPackageInfo[] { // Parse the package.json content @@ -29,14 +28,24 @@ export class TypeScriptPackageCrawler implements PackageCrawler { const dependencies = Object.entries(jsonData.dependencies || {}).concat( Object.entries(jsonData.devDependencies || {}), ); - const filtered = dependencies.filter( - ([name, version]) => !name.startsWith("@types/"), - ); + + // Filter out types packages and check if typescript is present + let foundTypes = false; + const filtered = dependencies.filter(([name, _]) => { + if (name.startsWith("@types/")) { + foundTypes = true; + return false; + } + if (name.includes("typescript")) { + foundTypes = true; + } + return true; + }); return filtered.map(([name, version]) => ({ name, version, packageFile: file, - language: this.language, + language: foundTypes ? "ts" : "js", })); } diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx index 7dcc478fc6..ca6a8bceb1 100644 --- a/gui/src/components/dialogs/AddDocsDialog.tsx +++ b/gui/src/components/dialogs/AddDocsDialog.tsx @@ -2,8 +2,11 @@ import { CheckCircleIcon, ChevronDownIcon, ChevronUpIcon, + CodeBracketIcon, + InformationCircleIcon, + LinkIcon, } from "@heroicons/react/24/outline"; -import { SiteIndexingConfig } from "core"; +import { PackageDocsResult, SiteIndexingConfig } from "core"; import { usePostHog } from "posthog-js/react"; import { useContext, useLayoutEffect, useMemo, useRef, useState } from "react"; import { useDispatch, useSelector } from "react-redux"; @@ -16,6 +19,8 @@ import { import { RootState } from "../../redux/store"; import IndexingStatusViewer from "../indexing/IndexingStatus"; +import { ToolTip } from "../gui/Tooltip"; + function AddDocsDialog() { const posthog = usePostHog(); const dispatch = useDispatch(); @@ -67,72 +72,125 @@ function AddDocsDialog() { posthog.capture("add_docs_gui", { url: startUrl }); } - if (submittedConfig) { - const status = indexingStatuses[submittedConfig.startUrl]; - return ( - <div className="flex flex-col p-4"> - <div className="flex flex-row items-center gap-2"> - <CheckCircleIcon className="h-8 w-8" /> - <h1>{`Docs added`}</h1> - </div> - <div className="flex flex-col gap-1 text-stone-500"> - <p className="m-0 p-0">Title: {submittedConfig.title}</p> - <p className="m-0 p-0">Start URL: {submittedConfig.startUrl}</p> - {submittedConfig.rootUrl && ( - <p className="m-0 p-0">Root URL: {submittedConfig.rootUrl}</p> - )} - {submittedConfig.maxDepth && ( - <p className="m-0 p-0">Max depth: {submittedConfig.maxDepth}</p> - )} - {submittedConfig.faviconUrl && ( - <p className="m-0 p-0">Favicon URL: {submittedConfig.faviconUrl}</p> - )} - </div> - {!!status && ( - <div className="mt-4 flex flex-col divide-x-0 divide-y-2 divide-solid divide-zinc-700"> - <p className="m-0 mb-5 p-0 leading-snug">{`Type "@docs" and select ${submittedConfig.title} to reference these docs once indexing is complete. Check indexing status from the "More" page.`}</p> - <div className="pt-1"> - <IndexingStatusViewer status={status} /> - </div> - </div> - )} - <div className="mt-4 flex flex-row items-center justify-end gap-4"> - <SecondaryButton - className="" - onClick={() => { - setSubmittedConfig(undefined); - }} - > - Add another - </SecondaryButton> - <Button - className="" - onClick={() => { - dispatch(setDialogMessage(undefined)); - dispatch(setShowDialog(false)); - }} - > - Done - </Button> - </div> - </div> - ); - } + const handleSelectSuggestion = (docsResult: PackageDocsResult) => {}; + + // if (submittedConfig) { + // const status = indexingStatuses[submittedConfig.startUrl]; + // return ( + // <div className="flex flex-col p-4"> + // <div className="flex flex-row items-center gap-2"> + // <CheckCircleIcon className="h-8 w-8" /> + // <h1>{`Docs added`}</h1> + // </div> + // <div className="flex flex-col gap-1 text-stone-500"> + // <p className="m-0 p-0">Title: {submittedConfig.title}</p> + // <p className="m-0 p-0">Start URL: {submittedConfig.startUrl}</p> + // {submittedConfig.rootUrl && ( + // <p className="m-0 p-0">Root URL: {submittedConfig.rootUrl}</p> + // )} + // {submittedConfig.maxDepth && ( + // <p className="m-0 p-0">Max depth: {submittedConfig.maxDepth}</p> + // )} + // {submittedConfig.faviconUrl && ( + // <p className="m-0 p-0">Favicon URL: {submittedConfig.faviconUrl}</p> + // )} + // </div> + // {!!status && ( + // <div className="mt-4 flex flex-col divide-x-0 divide-y-2 divide-solid divide-zinc-700"> + // <p className="m-0 mb-5 p-0 leading-snug">{`Type "@docs" and select ${submittedConfig.title} to reference these docs once indexing is complete. Check indexing status from the "More" page.`}</p> + // <div className="pt-1"> + // <IndexingStatusViewer status={status} /> + // </div> + // </div> + // )} + // <div className="mt-4 flex flex-row items-center justify-end gap-4"> + // <SecondaryButton + // className="" + // onClick={() => { + // setSubmittedConfig(undefined); + // }} + // > + // Add another + // </SecondaryButton> + // <Button + // className="" + // onClick={() => { + // dispatch(setDialogMessage(undefined)); + // dispatch(setShowDialog(false)); + // }} + // > + // Done + // </Button> + // </div> + // </div> + // ); + // } return ( <div className="p-4"> <div className="mb-8"> <h1>Add documentation</h1> - {/* {docsSuggestions.map((docsResult) => { - const { error, details } = docsResult; - const { language, name, version } = docsResult.packageInfo; - return ( - <div key={`${language}-${name}-${version}`}> - <p className="m-0 p-0">{`${name}`}</p> - <p className="m-0 p-0">{error ? "error" : details.docsLink}</p> - </div> - ); - })} */} + {docsSuggestions.length ? ( + <div className="no-scrollbar max-h-[300px] overflow-y-auto"> + <table className="border-collapse p-0"> + <thead className="bg-vsc-background sticky -top-1 font-bold"> + <tr className=""> + <td className="pr-1"> + <CodeBracketIcon className="h-3.5 w-3.5" /> + </td> + <td className="pr-1">Title</td> + {/* <td className="pr-1">Version</td> */} + <td className="pr-1">Start Link</td> + <td></td> + </tr> + </thead> + <tbody className="p-0"> + <tr className="whitespace-nowrap">Add docs</tr> + {docsSuggestions.map((docsResult) => { + const { error, details } = docsResult; + const { language, name, version } = docsResult.packageInfo; + const id = `${language}-${name}-${version}`; + return ( + <tr key={id} className="p-0"> + <td> + <input type="checkbox"></input> + </td> + <td>{name}</td> + {/* <td>{version}</td> */} + <td className=""> + {error ? ( + <span className="text-vsc-input-border italic"> + No docs link found + </span> + ) : ( + <span className="flex flex-row items-center gap-2"> + <div> + <LinkIcon className="h-2 w-2" /> + </div> + <p className="lines lines-1 m-0 p-0"> + {details.docsLink} + </p> + </span> + )} + </td> + <td> + <InformationCircleIcon + data-tooltip-id={id} + className="text-vsc-foreground-muted h-3.5 w-3.5 cursor-help" + /> + + <ToolTip id={id} place="bottom"> + <p className="m-0 p-0">{`Version: ${version}`}</p> + <p className="m-0 p-0">{`Found in ${docsResult.packageInfo.packageFile.path}`}</p> + </ToolTip> + </td> + </tr> + ); + })} + </tbody> + </table> + </div> + ) : null} <p> Continue pre-indexes many common documentation sites, but if there's one you don't see in the dropdown, enter the URL here. @@ -143,7 +201,7 @@ function AddDocsDialog() { so that you can ask questions. </p> </div> - + {/* <form onSubmit={onSubmit} className="flex flex-col space-y-4"> <label> Title @@ -219,7 +277,7 @@ function AddDocsDialog() { Submit </Button> </div> - </form> + </form> */} </div> ); } diff --git a/gui/src/components/index.ts b/gui/src/components/index.ts index e380f3ef6f..74f27a44a3 100644 --- a/gui/src/components/index.ts +++ b/gui/src/components/index.ts @@ -4,6 +4,10 @@ import { getFontSize, isJetBrains } from "../util"; export const VSC_INPUT_BACKGROUND_VAR = "--vscode-input-background"; export const VSC_BACKGROUND_VAR = "--vscode-sideBar-background"; export const VSC_FOREGROUND_VAR = "--vscode-editor-foreground"; +export const VSC_FOREGROUND_MUTED_VAR = "--vscode-foreground-muted"; +export const VSC_DESCRIPTION_FOREGROUND = "--vscode-descriptionForeground"; +export const VSC_INPUT_PLACEHOLDER_FOREGROUND = + "--vscode-input-placeholderForeground"; export const VSC_BUTTON_BACKGROUND_VAR = "--vscode-button-background"; export const VSC_BUTTON_FOREGROUND_VAR = "--vscode-button-foreground"; export const VSC_EDITOR_BACKGROUND_VAR = "--vscode-editor-background"; @@ -27,6 +31,9 @@ export const VSC_THEME_COLOR_VARS = [ VSC_INPUT_BACKGROUND_VAR, VSC_BACKGROUND_VAR, VSC_FOREGROUND_VAR, + VSC_FOREGROUND_MUTED_VAR, + VSC_DESCRIPTION_FOREGROUND, + VSC_INPUT_PLACEHOLDER_FOREGROUND, VSC_BUTTON_BACKGROUND_VAR, VSC_BUTTON_FOREGROUND_VAR, VSC_EDITOR_BACKGROUND_VAR, @@ -51,6 +58,9 @@ export const vscInputBackground = `var(${VSC_INPUT_BACKGROUND_VAR}, rgb(45 45 45 export const vscQuickInputBackground = `var(${VSC_QUICK_INPUT_BACKGROUND_VAR}, ${VSC_INPUT_BACKGROUND_VAR}, rgb(45 45 45))`; export const vscBackground = `var(${VSC_BACKGROUND_VAR}, rgb(30 30 30))`; export const vscForeground = `var(${VSC_FOREGROUND_VAR}, #fff)`; +export const vscForegroundMuted = `var(${VSC_FOREGROUND_MUTED_VAR}, #999)`; +export const vscDescriptionForeground = `var(${VSC_DESCRIPTION_FOREGROUND}, #999)`; +export const vscInputPlaceholderForeground = `var(${VSC_INPUT_PLACEHOLDER_FOREGROUND}, #999)`; export const vscButtonBackground = `var(${VSC_BUTTON_BACKGROUND_VAR}, #1bbe84)`; export const vscButtonForeground = `var(${VSC_BUTTON_FOREGROUND_VAR}, #ffffff)`; export const vscEditorBackground = `var(${VSC_EDITOR_BACKGROUND_VAR}, ${VSC_BACKGROUND_VAR}, rgb(30 30 30))`; diff --git a/gui/src/components/indexing/IndexingStatuses.tsx b/gui/src/components/indexing/IndexingStatuses.tsx index 7441212f91..61bf102a98 100644 --- a/gui/src/components/indexing/IndexingStatuses.tsx +++ b/gui/src/components/indexing/IndexingStatuses.tsx @@ -34,20 +34,19 @@ function IndexingStatuses() { Manage your documentation sources </span> {/* <div className="flex max-h-[170px] flex-col gap-1 overflow-x-hidden overflow-y-scroll pr-2"> */} - {docsStatuses.length ? ( - docsStatuses.map((status) => { - return <IndexingStatusViewer key={status.id} status={status} />; - }) - ) : ( - <SecondaryButton - onClick={() => { - dispatch(setShowDialog(true)); - dispatch(setDialogMessage(<AddDocsDialog />)); - }} - > - Add Docs - </SecondaryButton> - )} + {docsStatuses.length + ? docsStatuses.map((status) => { + return <IndexingStatusViewer key={status.id} status={status} />; + }) + : null} + <SecondaryButton + onClick={() => { + dispatch(setShowDialog(true)); + dispatch(setDialogMessage(<AddDocsDialog />)); + }} + > + Add Docs + </SecondaryButton> </div> // </div> ); diff --git a/gui/tailwind.config.cjs b/gui/tailwind.config.cjs index f021c748b8..3b3642d6bf 100644 --- a/gui/tailwind.config.cjs +++ b/gui/tailwind.config.cjs @@ -46,8 +46,15 @@ module.exports = { "vsc-badge-background": "var(--vscode-badge-background, #1bbe84)", "vsc-badge-foreground": "var(--vscode-badge-foreground, #fff)", "vsc-sidebar-border": "var(--vscode-sideBar-border, transparent)", - "vsc-find-match": "var(--vscode-editor-findMatchBackground, rgba(255, 255, 0, 0.6))", - "vsc-find-match-selected": "var(--vscode-editor-findMatchHighlightBackground, rgba(255, 223, 0, 0.8))", + "vsc-find-match": + "var(--vscode-editor-findMatchBackground, rgba(255, 255, 0, 0.6))", + "vsc-find-match-selected": + "var(--vscode-editor-findMatchHighlightBackground, rgba(255, 223, 0, 0.8))", + "vsc-foreground-muted": "var(--vscode-foreground-muted, #999)", + "vsc-description-foreground": + "var(--vscode-descriptionForeground, #999)", + "vsc-input-placeholder-foreground": + "var(--vscode-input-placeholderForeground, #999)", }, }, }, From 504c0c3843e5388fcd661f9755be9b174e9ef334 Mon Sep 17 00:00:00 2001 From: Dallin Romney <dallinromney@gmail.com> Date: Fri, 22 Nov 2024 02:51:34 -0800 Subject: [PATCH 06/17] autodetect docs v1 --- core/config/ProfileLifecycleManager.ts | 2 - core/config/load.ts | 1 + core/core.ts | 6 + core/index.d.ts | 2 + core/indexing/docs/suggestions/index.ts | 39 +- .../suggestions/packageCrawlers/Python.ts | 2 +- core/protocol/core.ts | 1 - core/protocol/webview.ts | 4 +- gui/src/components/dialogs/AddDocsDialog.tsx | 469 ++++++++++-------- gui/src/components/dialogs/index.tsx | 2 +- .../components/indexing/ChatIndexingPeeks.tsx | 2 +- .../components/indexing/DocsIndexingPeeks.tsx | 76 +++ .../components/indexing/IndexingStatuses.tsx | 41 +- 13 files changed, 391 insertions(+), 256 deletions(-) create mode 100644 gui/src/components/indexing/DocsIndexingPeeks.tsx diff --git a/core/config/ProfileLifecycleManager.ts b/core/config/ProfileLifecycleManager.ts index 85116f2133..8d44c8bac9 100644 --- a/core/config/ProfileLifecycleManager.ts +++ b/core/config/ProfileLifecycleManager.ts @@ -1,5 +1,3 @@ -import { config } from "dotenv"; - import { BrowserSerializedContinueConfig, ContinueConfig, diff --git a/core/config/load.ts b/core/config/load.ts index b692b36bf2..9911c840fe 100644 --- a/core/config/load.ts +++ b/core/config/load.ts @@ -527,6 +527,7 @@ function finalToBrowserConfig( embeddingsProvider: final.embeddingsProvider?.id, ui: final.ui, experimental: final.experimental, + docs: final.docs, }; } diff --git a/core/core.ts b/core/core.ts index b2be3478c3..331ffbe857 100644 --- a/core/core.ts +++ b/core/core.ts @@ -735,6 +735,10 @@ export class Core { return this.docsService.initStatuses(); }); on("docs/getSuggestedDocs", async (msg) => { + if (hasRequestedDocs) { + return; + } // TODO, remove, hack because of rerendering + hasRequestedDocs = true; const suggestedDocs = await getAllSuggestedDocs(this.ide); this.messenger.send("docs/suggestions", suggestedDocs); }); @@ -842,3 +846,5 @@ export class Core { // private } + +let hasRequestedDocs = false; diff --git a/core/index.d.ts b/core/index.d.ts index 5541584ebd..35c072bdb2 100644 --- a/core/index.d.ts +++ b/core/index.d.ts @@ -1189,6 +1189,7 @@ export interface BrowserSerializedContinueConfig { reranker?: RerankerDescription; experimental?: ExperimentalConfig; analytics?: AnalyticsConfig; + docs?: SiteIndexingConfig[]; } // DOCS SUGGESTIONS AND PACKAGE INFO @@ -1210,6 +1211,7 @@ export type ParsedPackageInfo = { export type PackageDetails = { docsLink?: string; + docsLinkWarning?: string; title?: string; description?: string; repo?: string; diff --git a/core/indexing/docs/suggestions/index.ts b/core/indexing/docs/suggestions/index.ts index 4e0c2104ac..db192489ad 100644 --- a/core/indexing/docs/suggestions/index.ts +++ b/core/indexing/docs/suggestions/index.ts @@ -63,9 +63,10 @@ export async function getAllSuggestedDocs(ide: IDE) { ); // Parse package files and build map of language -> packages - const packagesByRegistry: Record<string, ParsedPackageInfo[]> = {}; + const packagesByCrawler: Record<string, ParsedPackageInfo[]> = {}; PACKAGE_CRAWLERS.forEach((Crawler) => { const crawler = new Crawler(); + packagesByCrawler[crawler.packageRegistry] = []; const packageFiles = packageFilesByRegistry[crawler.packageRegistry]; packageFiles.forEach((file) => { const contents = fileContents.get(file.path); @@ -73,10 +74,7 @@ export async function getAllSuggestedDocs(ide: IDE) { return; } const packages = crawler.parsePackageFile(file, contents); - if (!packagesByRegistry[crawler.packageRegistry]) { - packagesByRegistry[crawler.packageRegistry] = []; - } - packagesByRegistry[crawler.packageRegistry].push(...packages); + packagesByCrawler[crawler.packageRegistry].push(...packages); }); }); @@ -84,13 +82,13 @@ export async function getAllSuggestedDocs(ide: IDE) { // TODO - this is where you would allow docs for different versions // by e.g. using "name-version" as the map key instead of just name // For now have not allowed - const languages = Object.keys(packagesByRegistry); - languages.forEach((language) => { - const packages = packagesByRegistry[language]; + const registries = Object.keys(packagesByCrawler); + registries.forEach((registry) => { + const packages = packagesByCrawler[registry]; const uniquePackages = Array.from( new Map(packages.map((pkg) => [pkg.name, pkg])).values(), ); - packagesByRegistry[language] = uniquePackages; + packagesByCrawler[registry] = uniquePackages; }); // Get documentation links for all packages @@ -98,7 +96,7 @@ export async function getAllSuggestedDocs(ide: IDE) { await Promise.all( PACKAGE_CRAWLERS.map(async (Crawler) => { const crawler = new Crawler(); - const packages = packagesByRegistry[crawler.packageRegistry]; + const packages = packagesByCrawler[crawler.packageRegistry]; const docsByRegistry = await Promise.all( packages.map(async (packageInfo) => { try { @@ -114,6 +112,11 @@ export async function getAllSuggestedDocs(ide: IDE) { details: { ...details, docsLink: details.docsLink, + docsLinkWarning: details.docsLink.includes("github.com") + ? "Github docs not supported, find the docs site" + : details.docsLink.includes("docs") + ? undefined + : "May not be a docs site, check the URL", }, }; } catch (error) { @@ -129,19 +132,3 @@ export async function getAllSuggestedDocs(ide: IDE) { ); return allDocsResults; } - -// write me an interface PackageCrawler that contains: -// 1. property `language` to store a given language like "python" or "typescript" -// 2. has a method `getPackageFiles` which takes a list of file names and decides which ones match package/dependency files (e.g. package.json for typescript, requirements.txt for python, etc) -// 3. has a method `parsePackageFile` which returns a list of package name and version from a relevant package file, in a standardized format like semver -// 4. has a method `getDocumentationLink` to check for documentation link for a given package (e.g. GET `https://registry.npmjs.org/<package>` and find docs field for typescript, documentation link in the package metadata for PyPi, etc.) -// Then, write typescript classes to implement this typescript interface for the languages "python" and "typescript" - -// I want to present the user with a list of dependencies and allow them to select which ones to index (embed) documentation for. -// In order to prevent duplicate file reads, the process will be like this: -// 1. take in a list of filepaths called `filepaths` -// 2. loop an array of PackageCrawler classes to build a map of `language` (string) to `packageFilePaths` (string[]) -// 3. Get unique filepaths from `packageFilePaths` and build a map ` of filepath to file contents using an existing `readFile` function, and skipping file reads of already in the map -// Finally, -// Add a `` method to the interface and classes that returns -// Then, assemble the classes in an array, and write a function getAllSuggestedDocs that returns a map of `language` to an ar diff --git a/core/indexing/docs/suggestions/packageCrawlers/Python.ts b/core/indexing/docs/suggestions/packageCrawlers/Python.ts index d00045f976..6a486c9849 100644 --- a/core/indexing/docs/suggestions/packageCrawlers/Python.ts +++ b/core/indexing/docs/suggestions/packageCrawlers/Python.ts @@ -30,7 +30,7 @@ export class PythonPackageCrawler implements PackageCrawler { .split("\n") .map((line) => { const [name, version] = line.split("=="); - return { name, version, packageFile: file, language: "python" }; + return { name, version, packageFile: file, language: "py" }; }) .filter((pkg) => pkg.name && pkg.version); } diff --git a/core/protocol/core.ts b/core/protocol/core.ts index 069f9e297a..f5b04c9cf5 100644 --- a/core/protocol/core.ts +++ b/core/protocol/core.ts @@ -9,7 +9,6 @@ import type { DiffLine, FileSymbolMap, IdeSettings, - IndexingStatusMap, LLMFullCompletionOptions, MessageContent, ModelDescription, diff --git a/core/protocol/webview.ts b/core/protocol/webview.ts index bf90497c8a..fb6a8837c5 100644 --- a/core/protocol/webview.ts +++ b/core/protocol/webview.ts @@ -2,9 +2,9 @@ import { ConfigValidationError } from "../config/validation.js"; import type { ContextItemWithId, - DocsSuggestions, IndexingProgressUpdate, IndexingStatus, + PackageDocsResult, } from "../index.js"; export type ToWebviewFromIdeOrCoreProtocol = { @@ -26,5 +26,5 @@ export type ToWebviewFromIdeOrCoreProtocol = { getWebviewHistoryLength: [undefined, number]; signInToControlPlane: [undefined, void]; openDialogMessage: ["account", void]; - "docs/suggestions": [DocsSuggestions, void]; + "docs/suggestions": [PackageDocsResult[], void]; }; diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx index ca6a8bceb1..0a90a60d22 100644 --- a/gui/src/components/dialogs/AddDocsDialog.tsx +++ b/gui/src/components/dialogs/AddDocsDialog.tsx @@ -1,12 +1,15 @@ import { CheckCircleIcon, + CheckIcon, ChevronDownIcon, ChevronUpIcon, CodeBracketIcon, + ExclamationTriangleIcon, InformationCircleIcon, LinkIcon, + PlusCircleIcon, } from "@heroicons/react/24/outline"; -import { PackageDocsResult, SiteIndexingConfig } from "core"; +import { IndexingStatus, PackageDocsResult, SiteIndexingConfig } from "core"; import { usePostHog } from "posthog-js/react"; import { useContext, useLayoutEffect, useMemo, useRef, useState } from "react"; import { useDispatch, useSelector } from "react-redux"; @@ -20,39 +23,79 @@ import { RootState } from "../../redux/store"; import IndexingStatusViewer from "../indexing/IndexingStatus"; import { ToolTip } from "../gui/Tooltip"; +import FileIcon from "../FileIcon"; +import DocsIndexingPeeks from "../indexing/DocsIndexingPeeks"; +import { updateIndexingStatus } from "../../redux/slices/stateSlice"; +import preIndexedDocs from "core/indexing/docs/preIndexedDocs"; function AddDocsDialog() { const posthog = usePostHog(); const dispatch = useDispatch(); - const ref = useRef<HTMLInputElement>(null); + const titleRef = useRef<HTMLInputElement>(null); + const urlRef = useRef<HTMLInputElement>(null); const [title, setTitle] = useState(""); const [startUrl, setStartUrl] = useState(""); const [faviconUrl, setFaviconUrl] = useState(""); - const [isOpen, setIsOpen] = useState(false); - - const [submittedConfig, setSubmittedConfig] = useState<SiteIndexingConfig>(); const ideMessenger = useContext(IdeMessengerContext); const indexingStatuses = useSelector( (store: RootState) => store.state.indexing.statuses, ); + const docsIndexingStatuses: IndexingStatus[] = useMemo(() => { + return Object.values(indexingStatuses).filter( + (status) => status.type === "docs" && status.status === "indexing", + ); + }, [indexingStatuses]); + const docsSuggestions = useSelector( (store: RootState) => store.state.docsSuggestions, ); + const configDocs = useSelector((store: RootState) => store.state.config.docs); + + const sortedDocsSuggestions = useMemo(() => { + const docsFromConfig = configDocs ?? []; + const filtered = docsSuggestions.filter((sug) => { + const startUrl = sug.details?.docsLink; + return ( + !docsFromConfig.find((d) => d.startUrl === startUrl) && + !docsIndexingStatuses.find((s) => s.id === startUrl) && + (startUrl ? !preIndexedDocs[startUrl] : true) + ); + }); + + filtered.sort((a, b) => { + const rank = (result: PackageDocsResult) => { + if (result.error) { + return 2; + } else if (result.details?.docsLinkWarning) { + return 1; + } else { + return 0; + } + }; + return rank(a) - rank(b); + }); + return filtered; + }, [docsSuggestions, configDocs, docsIndexingStatuses]); + const isFormValid = startUrl && title; useLayoutEffect(() => { setTimeout(() => { - if (ref.current) { - ref.current.focus(); + if (titleRef.current) { + titleRef.current.focus(); } }, 100); - }, [ref]); + }, [titleRef]); + const closeDialog = () => { + dispatch(setShowDialog(false)); + dispatch(setDialogMessage(undefined)); + }; function onSubmit(e: any) { e.preventDefault(); @@ -64,220 +107,236 @@ function AddDocsDialog() { ideMessenger.post("context/addDocs", siteIndexingConfig); - setSubmittedConfig(siteIndexingConfig); setTitle(""); setStartUrl(""); setFaviconUrl(""); posthog.capture("add_docs_gui", { url: startUrl }); + + // Optimistic status update + dispatch( + updateIndexingStatus({ + type: "docs", + description: "Initializing", + id: startUrl, + embeddingsProviderId: "mock-embeddings-provider-id", + progress: 0, + status: "indexing", + title, + url: startUrl, + }), + ); } - const handleSelectSuggestion = (docsResult: PackageDocsResult) => {}; + const handleSelectSuggestion = (docsResult: PackageDocsResult) => { + if (docsResult.error) { + setTitle(docsResult.packageInfo.name); + setStartUrl(""); + urlRef.current?.focus(); + return; + } + if (docsResult.details?.docsLinkWarning) { + setTitle(docsResult.packageInfo.name); + setStartUrl(docsResult.details.docsLink); + urlRef.current?.focus(); + return; + } + const siteIndexingConfig: SiteIndexingConfig = { + startUrl: docsResult.details.docsLink, + title: docsResult.details.title, + faviconUrl: undefined, + }; + + ideMessenger.post("context/addDocs", siteIndexingConfig); + + posthog.capture("add_docs_gui", { url: startUrl }); - // if (submittedConfig) { - // const status = indexingStatuses[submittedConfig.startUrl]; - // return ( - // <div className="flex flex-col p-4"> - // <div className="flex flex-row items-center gap-2"> - // <CheckCircleIcon className="h-8 w-8" /> - // <h1>{`Docs added`}</h1> - // </div> - // <div className="flex flex-col gap-1 text-stone-500"> - // <p className="m-0 p-0">Title: {submittedConfig.title}</p> - // <p className="m-0 p-0">Start URL: {submittedConfig.startUrl}</p> - // {submittedConfig.rootUrl && ( - // <p className="m-0 p-0">Root URL: {submittedConfig.rootUrl}</p> - // )} - // {submittedConfig.maxDepth && ( - // <p className="m-0 p-0">Max depth: {submittedConfig.maxDepth}</p> - // )} - // {submittedConfig.faviconUrl && ( - // <p className="m-0 p-0">Favicon URL: {submittedConfig.faviconUrl}</p> - // )} - // </div> - // {!!status && ( - // <div className="mt-4 flex flex-col divide-x-0 divide-y-2 divide-solid divide-zinc-700"> - // <p className="m-0 mb-5 p-0 leading-snug">{`Type "@docs" and select ${submittedConfig.title} to reference these docs once indexing is complete. Check indexing status from the "More" page.`}</p> - // <div className="pt-1"> - // <IndexingStatusViewer status={status} /> - // </div> - // </div> - // )} - // <div className="mt-4 flex flex-row items-center justify-end gap-4"> - // <SecondaryButton - // className="" - // onClick={() => { - // setSubmittedConfig(undefined); - // }} - // > - // Add another - // </SecondaryButton> - // <Button - // className="" - // onClick={() => { - // dispatch(setDialogMessage(undefined)); - // dispatch(setShowDialog(false)); - // }} - // > - // Done - // </Button> - // </div> - // </div> - // ); - // } + // Optimistic status update + dispatch( + updateIndexingStatus({ + type: "docs", + description: "Initializing", + id: docsResult.details.docsLink, + embeddingsProviderId: "mock-embeddings-provider-id", + progress: 0, + status: "indexing", + title: docsResult.details.title ?? docsResult.packageInfo.name, + url: docsResult.details.docsLink, + }), + ); + }; return ( - <div className="p-4"> - <div className="mb-8"> - <h1>Add documentation</h1> - {docsSuggestions.length ? ( - <div className="no-scrollbar max-h-[300px] overflow-y-auto"> - <table className="border-collapse p-0"> - <thead className="bg-vsc-background sticky -top-1 font-bold"> - <tr className=""> - <td className="pr-1"> - <CodeBracketIcon className="h-3.5 w-3.5" /> - </td> - <td className="pr-1">Title</td> - {/* <td className="pr-1">Version</td> */} - <td className="pr-1">Start Link</td> - <td></td> - </tr> - </thead> - <tbody className="p-0"> - <tr className="whitespace-nowrap">Add docs</tr> - {docsSuggestions.map((docsResult) => { - const { error, details } = docsResult; - const { language, name, version } = docsResult.packageInfo; - const id = `${language}-${name}-${version}`; - return ( - <tr key={id} className="p-0"> - <td> - <input type="checkbox"></input> - </td> - <td>{name}</td> - {/* <td>{version}</td> */} - <td className=""> - {error ? ( - <span className="text-vsc-input-border italic"> - No docs link found - </span> - ) : ( - <span className="flex flex-row items-center gap-2"> - <div> - <LinkIcon className="h-2 w-2" /> - </div> - <p className="lines lines-1 m-0 p-0"> - {details.docsLink} - </p> - </span> - )} - </td> - <td> - <InformationCircleIcon - data-tooltip-id={id} - className="text-vsc-foreground-muted h-3.5 w-3.5 cursor-help" + <div className="px-2 py-4 sm:px-4"> + <div className=""> + <h1 className="mb-0 hidden sm:block">Add documentation</h1> + <h1 className="sm:hidden">Add docs</h1> + {sortedDocsSuggestions.length && ( + <p className="m-0 mb-1 mt-4 p-0 font-semibold">Suggestions</p> + )} + <div className="border-vsc-foreground-muted max-h-[175px] overflow-y-scroll rounded-sm py-1 pr-2"> + {sortedDocsSuggestions.map((docsResult) => { + const { error, details } = docsResult; + const { language, name, version } = docsResult.packageInfo; + const id = `${language}-${name}-${version}`; + return ( + <> + <div + key={id} + className="grid cursor-pointer grid-cols-[auto_minmax(0,1fr)_minmax(0,1fr)_auto] items-center px-1 py-1 hover:bg-gray-200/10" + onClick={() => { + handleSelectSuggestion(docsResult); + }} + > + <div className="pr-1"> + {error ? ( + <div> + <ExclamationTriangleIcon + data-tooltip-id={id + "-error"} + className="h-4 w-4 text-red-500" /> - - <ToolTip id={id} place="bottom"> - <p className="m-0 p-0">{`Version: ${version}`}</p> - <p className="m-0 p-0">{`Found in ${docsResult.packageInfo.packageFile.path}`}</p> + <ToolTip id={id + "-error"} place="bottom"> + Docs URL not found </ToolTip> - </td> - </tr> - ); - })} - </tbody> - </table> - </div> - ) : null} - <p> - Continue pre-indexes many common documentation sites, but if there's - one you don't see in the dropdown, enter the URL here. - </p> - - <p> - Continue's indexing engine will crawl the site and generate embeddings - so that you can ask questions. - </p> - </div> - {/* - <form onSubmit={onSubmit} className="flex flex-col space-y-4"> - <label> - Title - <Input - type="text" - placeholder="Title" - value={title} - ref={ref} - onChange={(e) => setTitle(e.target.value)} - /> - <HelperText> - The title that will be displayed to users in the `@docs` submenu - </HelperText> - </label> + </div> + ) : details.docsLinkWarning ? ( + <div> + <ExclamationTriangleIcon + data-tooltip-id={id + "-warning"} + className="h-4 w-4 text-yellow-600" + /> + <ToolTip id={id + "-warning"} place="bottom"> + Start URL might not lead to docs + </ToolTip> + </div> + ) : ( + <PlusCircleIcon className="h-4 w-4 text-green-600" /> + )} + </div> + <div className="flex items-center gap-0.5"> + <div className="hidden sm:block"> + <FileIcon + filename={`x.${language}`} + height="1rem" + width="1rem" + /> + </div> + <span className="lines lines-1">{name}</span> + </div> + <div> + {error ? ( + <span className="text-vsc-input-border italic"> + No docs link found + </span> + ) : ( + <div className="flex items-center gap-2"> + {/* <div> + <LinkIcon className="h-2 w-2" /> + </div> */} + <p className="lines lines-1 m-0 p-0"> + {details.docsLink} + </p> + </div> + )} + </div> + <div> + <InformationCircleIcon + data-tooltip-id={id + "-info"} + className="text-vsc-foreground-muted h-3.5 w-3.5 select-none" + /> + <ToolTip id={id + "-info"} place="bottom"> + <p className="m-0 p-0">{`Version: ${version}`}</p> + <p className="m-0 p-0">{`Found in ${docsResult.packageInfo.packageFile.path}`}</p> + </ToolTip> + </div> + </div> + <ToolTip id={id} place="bottom"> + {error ? "Add to form" : "Index these docs"} + </ToolTip> + </> + ); + })} + </div> + <div className="mt-3"> + <form onSubmit={onSubmit} className="flex flex-col gap-1"> + <div className="flex flex-row gap-2"> + <label className="flex min-w-16 basis-1/4 flex-col gap-1"> + <div className="flex flex-row items-center gap-1"> + <span>Title</span> + <div> + <InformationCircleIcon + data-tooltip-id={"add-docs-form-title"} + className="text-vsc-foreground-muted h-3.5 w-3.5 select-none" + /> + <ToolTip id={"add-docs-form-title"} place="top"> + The title that will be displayed to users in the `@docs` + submenu + </ToolTip> + </div> + </div> - <label> - Start URL - <Input - type="url" - placeholder="Start URL" - value={startUrl} - onChange={(e) => { - setStartUrl(e.target.value); - }} - /> - <HelperText> - The starting location to begin crawling the documentation site - </HelperText> - </label> + <Input + type="text" + placeholder="Title" + value={title} + ref={titleRef} + onChange={(e) => setTitle(e.target.value)} + /> + </label> - <div - className="cursor-pointer" - onClick={() => setIsOpen((prev) => !prev)} - > - {isOpen ? ( - <ChevronUpIcon - width="1.0em" - height="1.0em" - style={{ color: lightGray }} - ></ChevronUpIcon> - ) : ( - <ChevronDownIcon - width="1.0em" - height="1.0em" - style={{ color: lightGray }} - ></ChevronDownIcon> - )} - <span className="ms-1">Advanced</span> + <label className="flex basis-3/4 flex-col gap-1"> + <div className="flex flex-row items-center gap-1"> + <span className="lines lines-1 whitespace-nowrap"> + Start URL + </span> + <div> + <InformationCircleIcon + data-tooltip-id={"add-docs-form-url"} + className="text-vsc-foreground-muted h-3.5 w-3.5 select-none" + /> + <ToolTip id={"add-docs-form-url"} place="top"> + The starting location to begin crawling the documentation + site + </ToolTip> + </div> + </div> + <Input + ref={urlRef} + type="url" + placeholder="Start URL" + value={startUrl} + onChange={(e) => { + setStartUrl(e.target.value); + }} + /> + </label> + </div> + <div className="flex flex-row justify-end gap-2"> + <SecondaryButton className="min-w-16" onClick={closeDialog}> + Done + </SecondaryButton> + <Button + className="min-w-16" + disabled={!isFormValid} + type="submit" + > + Go + </Button> + </div> + </form> </div> + </div> - {isOpen && ( - <div className="pt-2"> - <label> - Favicon URL [Optional] - <Input - type="url" - placeholder={`${startUrl}/favicon.ico`} - value={faviconUrl} - onChange={(e) => { - setFaviconUrl(e.target.value); - }} - /> - <HelperText> - The URL path to a favicon for the site - by default, it will be - `/favicon.ico` path from the Start URL - </HelperText> - </label> - </div> - )} - - <div className="flex justify-end"> - <Button disabled={!isFormValid} type="submit"> - Submit - </Button> + <DocsIndexingPeeks statuses={docsIndexingStatuses} /> + <div className="flex flex-row items-end justify-between gap-2"> + <div> + {docsIndexingStatuses.length ? ( + <p className="mt-2 p-0 text-xs text-stone-500"> + It is safe to close this form while indexing + </p> + ) : null} </div> - </form> */} + </div> </div> ); } diff --git a/gui/src/components/dialogs/index.tsx b/gui/src/components/dialogs/index.tsx index 7685109baf..8a3a67fe84 100644 --- a/gui/src/components/dialogs/index.tsx +++ b/gui/src/components/dialogs/index.tsx @@ -71,7 +71,7 @@ const TextDialog = (props: TextDialogProps) => { return ( <ScreenCover onClick={props.onClose} hidden={!props.showDialog}> <DialogContainer - className="xs:w-[85%] w-[92%] sm:w-[75%]" + className="xs:w-[90%] no-scrollbar max-h-full w-[92%] max-w-[600px] overflow-auto sm:w-[88%] md:w-[80%]" onClick={(e) => { e.stopPropagation(); }} diff --git a/gui/src/components/indexing/ChatIndexingPeeks.tsx b/gui/src/components/indexing/ChatIndexingPeeks.tsx index c77bdc323d..373689adc2 100644 --- a/gui/src/components/indexing/ChatIndexingPeeks.tsx +++ b/gui/src/components/indexing/ChatIndexingPeeks.tsx @@ -104,7 +104,7 @@ function ChatIndexingPeeks() { return ( <div className="flex flex-col gap-1"> {mergedIndexingStates.map((state) => { - return <ChatIndexingPeek state={state} />; + return <ChatIndexingPeek key={state.type} state={state} />; })} </div> ); diff --git a/gui/src/components/indexing/DocsIndexingPeeks.tsx b/gui/src/components/indexing/DocsIndexingPeeks.tsx new file mode 100644 index 0000000000..e5b98d42e7 --- /dev/null +++ b/gui/src/components/indexing/DocsIndexingPeeks.tsx @@ -0,0 +1,76 @@ +import { useDispatch, useSelector } from "react-redux"; +import { RootState } from "../../redux/store"; + +import { IndexingStatus } from "core"; +import { useMemo } from "react"; +import { useNavigate } from "react-router-dom"; +import { ArrowPathIcon, EyeSlashIcon } from "@heroicons/react/24/outline"; +import { + setDialogMessage, + setShowDialog, +} from "../../redux/slices/uiStateSlice"; + +export interface DocsIndexingPeekProps { + status: IndexingStatus; +} + +function DocsIndexingPeek({ status }: DocsIndexingPeekProps) { + const navigate = useNavigate(); + const dispatch = useDispatch(); + + const progressPercentage = useMemo(() => { + return Math.min(100, Math.max(0, status.progress * 100)); + }, [status.progress]); + + return ( + <div + className="flex cursor-pointer flex-row items-center gap-2 rounded-md px-1 text-stone-500 hover:bg-gray-700/10" + onClick={() => { + navigate("/more"); + dispatch(setShowDialog(false)); + dispatch(setDialogMessage(undefined)); + }} + > + <p className="m-0 p-0 text-stone-500 group-hover:underline"> + {status.title} + </p> + <div className="my-2 h-1.5 flex-1 rounded-md border border-solid border-gray-400"> + <div + className={`h-full rounded-lg bg-stone-500 transition-all duration-200 ease-in-out`} + style={{ + width: `${progressPercentage}%`, + }} + /> + </div> + <div className="xs:flex hidden flex-row items-center gap-1 text-stone-500"> + <span className="text-xs no-underline"> + {progressPercentage.toFixed(0)}% + </span> + <ArrowPathIcon + className={`animate-spin-slow inline-block h-4 w-4 text-stone-500`} + ></ArrowPathIcon> + </div> + </div> + ); +} + +interface DocsIndexingPeeksProps { + statuses: IndexingStatus[]; +} + +function DocsIndexingPeekList({ statuses }: DocsIndexingPeeksProps) { + if (!statuses.length) return null; + + return ( + <div className="flex flex-col"> + <p className="mx-0 my-1.5 p-0 text-stone-500">Currently Indexing</p> + <div className="max-h-[100px] overflow-y-auto pr-2"> + {statuses.map((status) => { + return <DocsIndexingPeek key={status.id} status={status} />; + })} + </div> + </div> + ); +} + +export default DocsIndexingPeekList; diff --git a/gui/src/components/indexing/IndexingStatuses.tsx b/gui/src/components/indexing/IndexingStatuses.tsx index 61bf102a98..746bc71ed9 100644 --- a/gui/src/components/indexing/IndexingStatuses.tsx +++ b/gui/src/components/indexing/IndexingStatuses.tsx @@ -8,6 +8,8 @@ import { setShowDialog, } from "../../redux/slices/uiStateSlice"; import AddDocsDialog from "../dialogs/AddDocsDialog"; +import { PlusCircleIcon } from "@heroicons/react/24/outline"; +import { ToolTip } from "../gui/Tooltip"; function IndexingStatuses() { const indexingStatuses = useSelector( @@ -25,30 +27,35 @@ function IndexingStatuses() { <div className="flex flex-col gap-1"> <div className="flex flex-row items-center justify-between"> <h3 className="mb-1 mt-0 text-xl">@docs indexes</h3> - {/* <div className="border-1 rounded-full border"> - <ChevronUpIcon className="h-8 w-8" /> - </div> */} - {/* TODO add some way to hide, scroll, etc. */} + {docsStatuses.length ? ( + <> + <PlusCircleIcon + data-tooltip-id={"more-add-docs-button"} + className="text-vsc-foreground-muted h-5 w-5 cursor-pointer focus:ring-0" + onClick={() => { + dispatch(setShowDialog(true)); + dispatch(setDialogMessage(<AddDocsDialog />)); + }} + /> + <ToolTip id={"more-add-docs-button"} place="top"> + Add Docs + </ToolTip> + </> + ) : null} </div> <span className="text-xs text-stone-500"> Manage your documentation sources </span> - {/* <div className="flex max-h-[170px] flex-col gap-1 overflow-x-hidden overflow-y-scroll pr-2"> */} - {docsStatuses.length - ? docsStatuses.map((status) => { + <div className="flex max-h-[170px] flex-col gap-1 overflow-x-hidden overflow-y-scroll pr-2"> + {docsStatuses.length ? ( + docsStatuses.map((status) => { return <IndexingStatusViewer key={status.id} status={status} />; }) - : null} - <SecondaryButton - onClick={() => { - dispatch(setShowDialog(true)); - dispatch(setDialogMessage(<AddDocsDialog />)); - }} - > - Add Docs - </SecondaryButton> + ) : ( + <SecondaryButton>Add Docs</SecondaryButton> + )} + </div> </div> - // </div> ); } From b41e601798407615b504a2a52ab287d144f4716a Mon Sep 17 00:00:00 2001 From: Dallin Romney <dallinromney@gmail.com> Date: Fri, 22 Nov 2024 03:05:58 -0800 Subject: [PATCH 07/17] suggested docs UI tweaks --- .../suggestions/packageCrawlers/Python.ts | 1 - extensions/vscode/package-lock.json | 2 +- gui/package-lock.json | 8 +------ gui/src/components/dialogs/AddDocsDialog.tsx | 24 ++++++++++--------- .../components/indexing/DocsIndexingPeeks.tsx | 9 +++---- 5 files changed, 20 insertions(+), 24 deletions(-) diff --git a/core/indexing/docs/suggestions/packageCrawlers/Python.ts b/core/indexing/docs/suggestions/packageCrawlers/Python.ts index 6a486c9849..3a617568f4 100644 --- a/core/indexing/docs/suggestions/packageCrawlers/Python.ts +++ b/core/indexing/docs/suggestions/packageCrawlers/Python.ts @@ -39,7 +39,6 @@ export class PythonPackageCrawler implements PackageCrawler { packageInfo: ParsedPackageInfo, ): Promise<PackageDetails> { // Fetch metadata from PyPI to find the documentation link - const response = await fetch( `https://pypi.org/pypi/${packageInfo.name}/json`, ); diff --git a/extensions/vscode/package-lock.json b/extensions/vscode/package-lock.json index b30a951dfc..8d680fb94a 100644 --- a/extensions/vscode/package-lock.json +++ b/extensions/vscode/package-lock.json @@ -109,7 +109,7 @@ "cheerio": "^1.0.0-rc.12", "commander": "^12.0.0", "comment-json": "^4.2.3", - "dbinfoz": "^0.11.0", + "dbinfoz": "^0.14.0", "diff": "^7.0.0", "dotenv": "^16.4.5", "fastest-levenshtein": "^1.0.16", diff --git a/gui/package-lock.json b/gui/package-lock.json index 3be234fb8d..1a9e4fe3ff 100644 --- a/gui/package-lock.json +++ b/gui/package-lock.json @@ -56,7 +56,6 @@ "socket.io-client": "^4.7.2", "styled-components": "^5.3.6", "table": "^6.8.1", - "tailwind-scrollbar-hide": "^1.1.7", "tippy.js": "^6.3.7", "unist-util-visit": "^5.0.0", "uuid": "^9.0.1", @@ -112,7 +111,7 @@ "cheerio": "^1.0.0-rc.12", "commander": "^12.0.0", "comment-json": "^4.2.3", - "dbinfoz": "^0.11.0", + "dbinfoz": "^0.14.0", "diff": "^7.0.0", "dotenv": "^16.4.5", "fastest-levenshtein": "^1.0.16", @@ -8984,11 +8983,6 @@ "node": ">=10.0.0" } }, - "node_modules/tailwind-scrollbar-hide": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/tailwind-scrollbar-hide/-/tailwind-scrollbar-hide-1.1.7.tgz", - "integrity": "sha512-X324n9OtpTmOMqEgDUEA/RgLrNfBF/jwJdctaPZDzB3mppxJk7TLIDmOreEDm1Bq4R9LSPu4Epf8VSdovNU+iA==" - }, "node_modules/tailwindcss": { "version": "3.4.12", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.12.tgz", diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx index 0a90a60d22..1467a3fa50 100644 --- a/gui/src/components/dialogs/AddDocsDialog.tsx +++ b/gui/src/components/dialogs/AddDocsDialog.tsx @@ -167,14 +167,17 @@ function AddDocsDialog() { }; return ( - <div className="px-2 py-4 sm:px-4"> - <div className=""> + <div className="px-2 pt-4 sm:px-4"> + <div className="mb-2"> <h1 className="mb-0 hidden sm:block">Add documentation</h1> <h1 className="sm:hidden">Add docs</h1> + <p className="m-0 mt-2 p-0 text-stone-500"> + For the @docs context provider + </p> {sortedDocsSuggestions.length && ( <p className="m-0 mb-1 mt-4 p-0 font-semibold">Suggestions</p> )} - <div className="border-vsc-foreground-muted max-h-[175px] overflow-y-scroll rounded-sm py-1 pr-2"> + <div className="border-vsc-foreground-muted max-h-[145px] overflow-y-scroll rounded-sm py-1 pr-2"> {sortedDocsSuggestions.map((docsResult) => { const { error, details } = docsResult; const { language, name, version } = docsResult.packageInfo; @@ -328,14 +331,13 @@ function AddDocsDialog() { </div> <DocsIndexingPeeks statuses={docsIndexingStatuses} /> - <div className="flex flex-row items-end justify-between gap-2"> - <div> - {docsIndexingStatuses.length ? ( - <p className="mt-2 p-0 text-xs text-stone-500"> - It is safe to close this form while indexing - </p> - ) : null} - </div> + <div className="flex flex-row items-end justify-start gap-2"> + {docsIndexingStatuses.length ? ( + <p className="mt-2 flex flex-row items-center gap-1 p-0 px-1 text-xs text-stone-500"> + <CheckIcon className="h-3 w-3" /> + It is safe to close this form while indexing + </p> + ) : null} </div> </div> ); diff --git a/gui/src/components/indexing/DocsIndexingPeeks.tsx b/gui/src/components/indexing/DocsIndexingPeeks.tsx index e5b98d42e7..9c5f96ec85 100644 --- a/gui/src/components/indexing/DocsIndexingPeeks.tsx +++ b/gui/src/components/indexing/DocsIndexingPeeks.tsx @@ -1,10 +1,9 @@ -import { useDispatch, useSelector } from "react-redux"; -import { RootState } from "../../redux/store"; +import { useDispatch } from "react-redux"; import { IndexingStatus } from "core"; import { useMemo } from "react"; import { useNavigate } from "react-router-dom"; -import { ArrowPathIcon, EyeSlashIcon } from "@heroicons/react/24/outline"; +import { ArrowPathIcon } from "@heroicons/react/24/outline"; import { setDialogMessage, setShowDialog, @@ -63,7 +62,9 @@ function DocsIndexingPeekList({ statuses }: DocsIndexingPeeksProps) { return ( <div className="flex flex-col"> - <p className="mx-0 my-1.5 p-0 text-stone-500">Currently Indexing</p> + <p className="mx-0 my-1.5 p-0 px-1 font-semibold text-stone-500"> + Currently Indexing + </p> <div className="max-h-[100px] overflow-y-auto pr-2"> {statuses.map((status) => { return <DocsIndexingPeek key={status.id} status={status} />; From 6cb9adc110e774e4686f99c1fbe7b1de2e438878 Mon Sep 17 00:00:00 2001 From: Dallin Romney <dallinromney@gmail.com> Date: Fri, 22 Nov 2024 03:10:45 -0800 Subject: [PATCH 08/17] suggested docs - cleanup --- gui/src/components/dialogs/SuggestedDocs.tsx | 18 ---- manual-testing-sandbox/package.json | 102 ------------------- manual-testing-sandbox/requirements.txt | 10 -- 3 files changed, 130 deletions(-) delete mode 100644 gui/src/components/dialogs/SuggestedDocs.tsx delete mode 100644 manual-testing-sandbox/package.json delete mode 100644 manual-testing-sandbox/requirements.txt diff --git a/gui/src/components/dialogs/SuggestedDocs.tsx b/gui/src/components/dialogs/SuggestedDocs.tsx deleted file mode 100644 index 1540ea92f9..0000000000 --- a/gui/src/components/dialogs/SuggestedDocs.tsx +++ /dev/null @@ -1,18 +0,0 @@ -import { PackageDocsResult } from "core"; - -interface SuggestedDocProps { - docResult: PackageDocsResult; -} -const SuggestedDoc = ({ docResult }: SuggestedDocProps) => { - return <div>SuggestedDoc</div>; -}; - -interface SuggestedDocsListProps { - docs: PackageDocsResult[]; - // on -} -const SuggestedDocsList = ({ docs }: SuggestedDocsListProps) => { - return <div></div>; -}; - -export default SuggestedDocsList; diff --git a/manual-testing-sandbox/package.json b/manual-testing-sandbox/package.json deleted file mode 100644 index ef77ac910c..0000000000 --- a/manual-testing-sandbox/package.json +++ /dev/null @@ -1,102 +0,0 @@ -{ - "name": "gui", - "private": true, - "type": "module", - "author": "Continue Dev, Inc", - "license": "Apache-2.0", - "scripts": { - "dev": "vite", - "tsc:check": "tsc -p ./ --noEmit", - "build": "tsc && vite build", - "preview": "vite preview", - "test": "vitest run", - "test:coverage": "vitest run --coverage", - "test:ui": "vitest --ui", - "test:watch": "vitest" - }, - "dependencies": { - "@headlessui/react": "^1.7.17", - "@heroicons/react": "^2.0.18", - "@monaco-editor/react": "^4.6.0", - "@reduxjs/toolkit": "^1.9.3", - "@tiptap/core": "^2.3.2", - "@tiptap/extension-document": "^2.3.2", - "@tiptap/extension-dropcursor": "^2.1.16", - "@tiptap/extension-history": "^2.3.2", - "@tiptap/extension-image": "^2.1.16", - "@tiptap/extension-mention": "^2.1.13", - "@tiptap/extension-paragraph": "^2.3.2", - "@tiptap/extension-placeholder": "^2.1.13", - "@tiptap/extension-text": "^2.3.2", - "@tiptap/pm": "^2.1.13", - "@tiptap/react": "^2.1.13", - "@tiptap/starter-kit": "^2.1.13", - "@tiptap/suggestion": "^2.1.13", - "@types/vscode-webview": "^1.57.1", - "core": "file:../core", - "dompurify": "^3.0.6", - "downshift": "^7.6.0", - "lodash": "^4.17.21", - "minisearch": "^7.0.2", - "onigasm": "^2.2.5", - "posthog-js": "^1.130.1", - "prismjs": "^1.29.0", - "react": "^18.2.0", - "react-dom": "^18.2.0", - "react-error-boundary": "^4.0.11", - "react-hook-form": "^7.47.0", - "react-intersection-observer": "^9.13.1", - "react-markdown": "^9.0.1", - "react-redux": "^8.0.5", - "react-remark": "^2.1.0", - "react-router-dom": "^6.14.2", - "react-switch": "^7.0.0", - "react-syntax-highlighter": "^15.5.0", - "react-tooltip": "^5.18.0", - "redux-persist": "^6.0.0", - "redux-persist-transform-filter": "^0.0.22", - "rehype-highlight": "^7.0.0", - "rehype-katex": "^7.0.0", - "rehype-wrap-all": "^1.1.0", - "remark-math": "^6.0.0", - "reselect": "^5.1.1", - "seti-file-icons": "^0.0.8", - "socket.io-client": "^4.7.2", - "styled-components": "^5.3.6", - "table": "^6.8.1", - "tippy.js": "^6.3.7", - "unist-util-visit": "^5.0.0", - "uuid": "^9.0.1", - "vscode-webview": "^1.0.1-beta.1" - }, - "devDependencies": { - "@swc/cli": "^0.3.14", - "@swc/core": "^1.7.26", - "@testing-library/dom": "^10.4.0", - "@testing-library/jest-dom": "^6.5.0", - "@testing-library/react": "^16.0.1", - "@testing-library/user-event": "^14.5.2", - "@types/lodash": "^4.17.6", - "@types/node": "^20.5.6", - "@types/node-fetch": "^2.6.4", - "@types/react": "^18.3.11", - "@types/react-dom": "^18.3.1", - "@types/react-router-dom": "^5.3.3", - "@types/react-syntax-highlighter": "^15.5.7", - "@types/styled-components": "^5.1.26", - "@vitejs/plugin-react-swc": "^3.7.0", - "@vitest/coverage-v8": "^2.1.3", - "@vitest/ui": "^2.1.3", - "autoprefixer": "^10.4.13", - "jsdom": "^25.0.1", - "postcss": "^8.4.21", - "tailwindcss": "^3.2.7", - "typescript": "^4.9.3", - "vite": "^4.1.0", - "vitest": "^2.1.3" - }, - "engine-strict": true, - "engines": { - "node": ">=20.11.0" - } -} diff --git a/manual-testing-sandbox/requirements.txt b/manual-testing-sandbox/requirements.txt deleted file mode 100644 index 793d6a8275..0000000000 --- a/manual-testing-sandbox/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -flask==2.1.1 -requests==2.28.1 -numpy==1.23.4 -pandas==1.5.0 -scipy==1.9.3 -django==4.1.3 -matplotlib==3.6.2 -pytest==7.2.0 -sqlalchemy==1.4.41 -beautifulsoup4==4.11.1 \ No newline at end of file From 0b0a0e644525f30f7617b95da0347e8042b64f39 Mon Sep 17 00:00:00 2001 From: Dallin Romney <dallinromney@gmail.com> Date: Fri, 22 Nov 2024 11:07:28 -0800 Subject: [PATCH 09/17] docs suggestions patrick bug fixes p1 --- manual-testing-sandbox/package.json | 45 +++++++++++++++++++ manual-testing-sandbox/requirements.txt | 57 +++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 manual-testing-sandbox/package.json create mode 100644 manual-testing-sandbox/requirements.txt diff --git a/manual-testing-sandbox/package.json b/manual-testing-sandbox/package.json new file mode 100644 index 0000000000..5bcec0c8fc --- /dev/null +++ b/manual-testing-sandbox/package.json @@ -0,0 +1,45 @@ +{ + "name": "my-react-project", + "version": "1.0.0", + "description": "A React project using Radix UI for accessible components", + "main": "index.js", + "scripts": { + "start": "react-scripts start", + "build": "react-scripts build", + "test": "react-scripts test", + "eject": "react-scripts eject" + }, + "keywords": [ + "react", + "radix-ui", + "accessibility", + "frontend" + ], + "author": "Your Name", + "license": "MIT", + "dependencies": { + "@radix-ui/react-accordion": "^1.0.0", + "@radix-ui/react-alert-dialog": "^1.0.0", + "@radix-ui/react-checkbox": "^1.0.0", + "react": "^18.0.0", + "react-dom": "^18.0.0", + "react-scripts": "^5.0.0" + }, + "devDependencies": { + "@types/react": "^18.0.0", + "@types/react-dom": "^18.0.0", + "typescript": "^4.5.0" + }, + "browserslist": { + "production": [ + ">0.2%", + "not dead", + "not op_mini all" + ], + "development": [ + "last 1 chrome version", + "last 1 firefox version", + "last 1 safari version" + ] + } +} diff --git a/manual-testing-sandbox/requirements.txt b/manual-testing-sandbox/requirements.txt new file mode 100644 index 0000000000..87f2e654bf --- /dev/null +++ b/manual-testing-sandbox/requirements.txt @@ -0,0 +1,57 @@ +# Core packages +numpy==1.21.2 +pandas==1.3.3 +scipy==1.7.1 + +# Web development +flask==2.0.2 +django==3.2.7 + +# Request handling +requests==2.26.0 +httpx==0.19.0 + +# Machine Learning +scikit-learn==0.24.2 +tensorflow==2.6.0 + +# Data visualization +matplotlib==3.4.3 +seaborn==0.11.2 +plotly==5.3.1 + +# Natural Language Processing +nltk==3.6.3 +spacy==3.1.2 + +# Deep Learning +torch==1.9.1 +keras==2.6.0 + +# Image processing +pillow==8.3.2 +opencv-python==4.5.3.56 + +# Data handling and manipulation +beautifulsoup4==4.10.0 +sqlalchemy==1.4.25 +xlrd==2.0.1 +openpyxl==3.0.9 + +# Utilities +pyyaml==5.4.1 +python-dotenv==0.19.0 + +# Testing +pytest==6.2.5 + +# Logging and Debugging +loguru==0.5.3 + +# Asynchronous Programming +aiohttp==3.7.4 + +# Others +jupyter==1.0.0 +gunicorn==20.1.0 +psycopg2==2.9.1 \ No newline at end of file From f4e9da5828b9f6040728f570692ddfd4f5634e55 Mon Sep 17 00:00:00 2001 From: Dallin Romney <dallinromney@gmail.com> Date: Fri, 22 Nov 2024 12:19:33 -0800 Subject: [PATCH 10/17] patrick tweaks --- .../docs/suggestions/packageCrawlers/TsJs.ts | 5 + gui/src/components/AccountDialog.tsx | 2 - gui/src/components/dialogs/AddDocsDialog.tsx | 159 +++++++++--------- .../components/indexing/DocsIndexingPeeks.tsx | 20 ++- .../components/indexing/IndexingStatuses.tsx | 37 ++-- 5 files changed, 115 insertions(+), 108 deletions(-) diff --git a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts index 42c5007480..3a9ef96c2b 100644 --- a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts +++ b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts @@ -59,6 +59,11 @@ export class NodePackageCrawler implements PackageCrawler { throw new Error(`Could not fetch data for package ${name}`); } const data = await response.json(); + + // const dependencies = Object.keys(packageContentData.dependencies || {}) + // .concat(Object.keys(packageContentData.devDependencies || {})); + // const usesTypescript = dependencies.includes("typescript"); + return { docsLink: data.homepage as string | undefined, title: name, // package.json doesn't have specific title field diff --git a/gui/src/components/AccountDialog.tsx b/gui/src/components/AccountDialog.tsx index 5438d11df6..db61fe3c0e 100644 --- a/gui/src/components/AccountDialog.tsx +++ b/gui/src/components/AccountDialog.tsx @@ -65,8 +65,6 @@ function AccountDialog() { ); } - const dispatch = useDispatch(); - const changeProfileId = (id: string) => { ideMessenger.post("didChangeSelectedProfile", { id }); dispatch(setSelectedProfileId(id)); diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx index 1467a3fa50..68039e85c3 100644 --- a/gui/src/components/dialogs/AddDocsDialog.tsx +++ b/gui/src/components/dialogs/AddDocsDialog.tsx @@ -7,7 +7,9 @@ import { ExclamationTriangleIcon, InformationCircleIcon, LinkIcon, + PencilIcon, PlusCircleIcon, + PlusIcon, } from "@heroicons/react/24/outline"; import { IndexingStatus, PackageDocsResult, SiteIndexingConfig } from "core"; import { usePostHog } from "posthog-js/react"; @@ -27,6 +29,7 @@ import FileIcon from "../FileIcon"; import DocsIndexingPeeks from "../indexing/DocsIndexingPeeks"; import { updateIndexingStatus } from "../../redux/slices/stateSlice"; import preIndexedDocs from "core/indexing/docs/preIndexedDocs"; +import { NewSessionButton } from "../mainInput/NewSessionButton"; function AddDocsDialog() { const posthog = usePostHog(); @@ -168,13 +171,13 @@ function AddDocsDialog() { return ( <div className="px-2 pt-4 sm:px-4"> - <div className="mb-2"> + <div className=""> <h1 className="mb-0 hidden sm:block">Add documentation</h1> <h1 className="sm:hidden">Add docs</h1> <p className="m-0 mt-2 p-0 text-stone-500"> For the @docs context provider </p> - {sortedDocsSuggestions.length && ( + {!!sortedDocsSuggestions.length && ( <p className="m-0 mb-1 mt-4 p-0 font-semibold">Suggestions</p> )} <div className="border-vsc-foreground-muted max-h-[145px] overflow-y-scroll rounded-sm py-1 pr-2"> @@ -183,80 +186,65 @@ function AddDocsDialog() { const { language, name, version } = docsResult.packageInfo; const id = `${language}-${name}-${version}`; return ( - <> - <div - key={id} - className="grid cursor-pointer grid-cols-[auto_minmax(0,1fr)_minmax(0,1fr)_auto] items-center px-1 py-1 hover:bg-gray-200/10" - onClick={() => { - handleSelectSuggestion(docsResult); - }} - > - <div className="pr-1"> - {error ? ( - <div> - <ExclamationTriangleIcon - data-tooltip-id={id + "-error"} - className="h-4 w-4 text-red-500" - /> - <ToolTip id={id + "-error"} place="bottom"> - Docs URL not found - </ToolTip> - </div> - ) : details.docsLinkWarning ? ( - <div> - <ExclamationTriangleIcon - data-tooltip-id={id + "-warning"} - className="h-4 w-4 text-yellow-600" - /> - <ToolTip id={id + "-warning"} place="bottom"> - Start URL might not lead to docs - </ToolTip> - </div> - ) : ( - <PlusCircleIcon className="h-4 w-4 text-green-600" /> - )} - </div> - <div className="flex items-center gap-0.5"> - <div className="hidden sm:block"> - <FileIcon - filename={`x.${language}`} - height="1rem" - width="1rem" + <div + key={id} + className="grid cursor-pointer grid-cols-[auto_minmax(0,1fr)_minmax(0,1fr)_auto] items-center px-1 py-1 hover:bg-gray-200/10" + onClick={() => { + handleSelectSuggestion(docsResult); + }} + > + <div className="pr-1"> + {error || details.docsLinkWarning ? ( + <div> + <PencilIcon + data-tooltip-id={id + "-edit"} + className="vsc-foreground-muted h-3 w-3" /> + <ToolTip id={id + "-edit"} place="bottom"> + This may not be a docs page + </ToolTip> </div> - <span className="lines lines-1">{name}</span> + ) : ( + <PlusIcon className="text-foreground-muted h-3.5 w-3.5" /> + )} + </div> + <div className="flex items-center gap-0.5"> + <div className="hidden sm:block"> + <FileIcon + filename={`x.${language}`} + height="1rem" + width="1rem" + /> </div> - <div> - {error ? ( - <span className="text-vsc-input-border italic"> - No docs link found - </span> - ) : ( - <div className="flex items-center gap-2"> - {/* <div> + <span className="lines lines-1">{name}</span> + </div> + <div> + {error ? ( + <span className="text-vsc-foreground-muted italic"> + No docs link found + </span> + ) : ( + <div className="flex items-center gap-2"> + {/* <div> <LinkIcon className="h-2 w-2" /> </div> */} - <p className="lines lines-1 m-0 p-0"> - {details.docsLink} - </p> - </div> - )} - </div> - <div> - <InformationCircleIcon - data-tooltip-id={id + "-info"} - className="text-vsc-foreground-muted h-3.5 w-3.5 select-none" - /> - <ToolTip id={id + "-info"} place="bottom"> - <p className="m-0 p-0">{`Version: ${version}`}</p> - <p className="m-0 p-0">{`Found in ${docsResult.packageInfo.packageFile.path}`}</p> - </ToolTip> - </div> + <p className="lines lines-1 m-0 p-0"> + {details.docsLink} + </p> + </div> + )} </div> - <ToolTip id={id} place="bottom"> - {error ? "Add to form" : "Index these docs"} - </ToolTip> - </> + <div> + <InformationCircleIcon + data-tooltip-id={id + "-info"} + className="text-vsc-foreground-muted h-3.5 w-3.5 select-none" + /> + <ToolTip id={id + "-info"} place="bottom"> + <p className="m-0 p-0">{`Version: ${version}`}</p> + <p className="m-0 p-0">{`Found in ${docsResult.packageInfo.packageFile.path}`}</p> + </ToolTip> + </div> + </div> ); })} </div> @@ -313,31 +301,36 @@ function AddDocsDialog() { }} /> </label> + {/* <div> + <PlusCircleIcon className="h-5 w-5 self-end" /> + </div> */} </div> <div className="flex flex-row justify-end gap-2"> - <SecondaryButton className="min-w-16" onClick={closeDialog}> - Done - </SecondaryButton> - <Button + <SecondaryButton className="min-w-16" disabled={!isFormValid} type="submit" > - Go - </Button> + Add + </SecondaryButton> </div> </form> </div> </div> <DocsIndexingPeeks statuses={docsIndexingStatuses} /> - <div className="flex flex-row items-end justify-start gap-2"> - {docsIndexingStatuses.length ? ( - <p className="mt-2 flex flex-row items-center gap-1 p-0 px-1 text-xs text-stone-500"> - <CheckIcon className="h-3 w-3" /> - It is safe to close this form while indexing - </p> - ) : null} + <div className="flex flex-row items-end justify-between pb-3"> + <div> + {docsIndexingStatuses.length ? ( + <p className="mt-2 flex flex-row items-center gap-1 p-0 px-1 text-xs text-stone-500"> + <CheckIcon className="h-3 w-3" /> + It is safe to close this form while indexing + </p> + ) : null} + </div> + {/* <Button className="min-w-16" onClick={closeDialog}> + Done + </Button> */} </div> </div> ); diff --git a/gui/src/components/indexing/DocsIndexingPeeks.tsx b/gui/src/components/indexing/DocsIndexingPeeks.tsx index 9c5f96ec85..e50b4adcf2 100644 --- a/gui/src/components/indexing/DocsIndexingPeeks.tsx +++ b/gui/src/components/indexing/DocsIndexingPeeks.tsx @@ -45,9 +45,9 @@ function DocsIndexingPeek({ status }: DocsIndexingPeekProps) { <span className="text-xs no-underline"> {progressPercentage.toFixed(0)}% </span> - <ArrowPathIcon + {/* <ArrowPathIcon className={`animate-spin-slow inline-block h-4 w-4 text-stone-500`} - ></ArrowPathIcon> + ></ArrowPathIcon> */} </div> </div> ); @@ -58,17 +58,19 @@ interface DocsIndexingPeeksProps { } function DocsIndexingPeekList({ statuses }: DocsIndexingPeeksProps) { - if (!statuses.length) return null; - return ( - <div className="flex flex-col"> + <div className="border-vsc-input-border mt-2 flex flex-col border-0 border-t border-solid pt-2"> <p className="mx-0 my-1.5 p-0 px-1 font-semibold text-stone-500"> - Currently Indexing + Currently Indexing: </p> <div className="max-h-[100px] overflow-y-auto pr-2"> - {statuses.map((status) => { - return <DocsIndexingPeek key={status.id} status={status} />; - })} + {statuses.length ? ( + statuses.map((status) => { + return <DocsIndexingPeek key={status.id} status={status} />; + }) + ) : ( + <p className="m-0 pl-1 font-semibold text-stone-500">None</p> + )} </div> </div> ); diff --git a/gui/src/components/indexing/IndexingStatuses.tsx b/gui/src/components/indexing/IndexingStatuses.tsx index 746bc71ed9..3ea04216f4 100644 --- a/gui/src/components/indexing/IndexingStatuses.tsx +++ b/gui/src/components/indexing/IndexingStatuses.tsx @@ -28,20 +28,29 @@ function IndexingStatuses() { <div className="flex flex-row items-center justify-between"> <h3 className="mb-1 mt-0 text-xl">@docs indexes</h3> {docsStatuses.length ? ( - <> - <PlusCircleIcon - data-tooltip-id={"more-add-docs-button"} - className="text-vsc-foreground-muted h-5 w-5 cursor-pointer focus:ring-0" - onClick={() => { - dispatch(setShowDialog(true)); - dispatch(setDialogMessage(<AddDocsDialog />)); - }} - /> - <ToolTip id={"more-add-docs-button"} place="top"> - Add Docs - </ToolTip> - </> - ) : null} + <div + className="border-vsc-foreground text-vsc-foreground enabled:hover:bg-vsc-background m-2 rounded border border-solid bg-inherit px-3 py-2 enabled:hover:cursor-pointer enabled:hover:opacity-90 disabled:text-gray-500" + onClick={() => { + dispatch(setShowDialog(true)); + dispatch(setDialogMessage(<AddDocsDialog />)); + }} + > + Add + </div> + ) : // <div> + // <PlusCircleIcon + // data-tooltip-id={"more-add-docs-button"} + // className="text-vsc-foreground-muted h-5 w-5 cursor-pointer focus:ring-0" + // onClick={() => { + // dispatch(setShowDialog(true)); + // dispatch(setDialogMessage(<AddDocsDialog />)); + // }} + // /> + // <ToolTip id={"more-add-docs-button"} place="top"> + // Add Docs + // </ToolTip> + // </div> + null} </div> <span className="text-xs text-stone-500"> Manage your documentation sources From 263c839153c5c9996f4699389d9b2594badf772d Mon Sep 17 00:00:00 2001 From: Dallin Romney <dallinromney@gmail.com> Date: Fri, 22 Nov 2024 15:41:41 -0800 Subject: [PATCH 11/17] docs - use config.docs instead of inferring from statuses --- core/core.ts | 6 +- core/index.d.ts | 2 - core/indexing/docs/DocsService.ts | 110 ++++++++++-------- core/protocol/core.ts | 3 +- core/protocol/passThrough.ts | 2 +- gui/src/App.tsx | 2 +- gui/src/components/AccountDialog.tsx | 2 + gui/src/components/dialogs/AddDocsDialog.tsx | 2 +- .../components/indexing/DocsIndexingPeeks.tsx | 6 +- ...exingStatus.tsx => DocsIndexingStatus.tsx} | 71 ++++++----- .../indexing/DocsIndexingStatuses.tsx | 57 +++++++++ .../components/indexing/IndexingStatuses.tsx | 71 ----------- gui/src/hooks/useSetup.ts | 7 +- gui/src/pages/More/More.tsx | 4 +- gui/src/redux/slices/stateSlice.ts | 2 +- .../{ => nested-folder}/package.json | 0 16 files changed, 179 insertions(+), 168 deletions(-) rename gui/src/components/indexing/{IndexingStatus.tsx => DocsIndexingStatus.tsx} (68%) create mode 100644 gui/src/components/indexing/DocsIndexingStatuses.tsx delete mode 100644 gui/src/components/indexing/IndexingStatuses.tsx rename manual-testing-sandbox/{ => nested-folder}/package.json (100%) diff --git a/core/core.ts b/core/core.ts index 331ffbe857..00ceabb4bd 100644 --- a/core/core.ts +++ b/core/core.ts @@ -731,9 +731,6 @@ export class Core { this.docsService.setPaused(msg.data.id, msg.data.paused); } }); - on("indexing/initStatuses", async (msg) => { - return this.docsService.initStatuses(); - }); on("docs/getSuggestedDocs", async (msg) => { if (hasRequestedDocs) { return; @@ -742,6 +739,9 @@ export class Core { const suggestedDocs = await getAllSuggestedDocs(this.ide); this.messenger.send("docs/suggestions", suggestedDocs); }); + on("docs/initStatuses", async (msg) => { + void this.docsService.initStatuses(); + }); // on("didChangeSelectedProfile", (msg) => { diff --git a/core/index.d.ts b/core/index.d.ts index 583e00a855..30aee52901 100644 --- a/core/index.d.ts +++ b/core/index.d.ts @@ -68,8 +68,6 @@ export interface IndexingStatus { url?: string; } -export type IndexingStatusMap = Map<string, IndexingStatus>; - export type PromptTemplateFunction = ( history: ChatMessage[], otherData: Record<string, string>, diff --git a/core/indexing/docs/DocsService.ts b/core/indexing/docs/DocsService.ts index 838e8fb542..f9c88cd0ed 100644 --- a/core/indexing/docs/DocsService.ts +++ b/core/indexing/docs/DocsService.ts @@ -7,7 +7,6 @@ import { ContinueConfig, EmbeddingsProvider, IDE, - IndexingStatusMap, IndexingStatus, SiteIndexingConfig, IdeInfo, @@ -86,7 +85,6 @@ export type AddParams = { export default class DocsService { static lanceTableName = "docs"; static sqlitebTableName = "docs"; - static indexingType = "docs"; static preIndexedDocsEmbeddingsProvider = new TransformersJsEmbeddingsProvider(); @@ -136,38 +134,55 @@ export default class DocsService { configHandler.onConfigUpdate(this.handleConfigUpdate.bind(this)); } - readonly statuses: IndexingStatusMap = new Map(); - - // Function for GUI to retrieve initial pending statuses - // And kickoff indexing where needed - async initStatuses() { - this.config?.docs?.forEach(async (doc) => { - const currentStatus = this.statuses.get(doc.startUrl); - if (currentStatus) { - this.handleStatusUpdate(currentStatus); - } else { - this.handleStatusUpdate({ - type: "docs", - id: doc.startUrl, - embeddingsProviderId: this.config.embeddingsProvider.id, - isReindexing: false, - progress: 0, - description: "Pending", - status: "pending", - title: doc.title, - debugInfo: `max depth: ${doc.maxDepth}`, - icon: doc.faviconUrl, - url: doc.startUrl, - }); - } - }); - } + readonly statuses: Map<string, IndexingStatus> = new Map(); handleStatusUpdate(update: IndexingStatus) { this.statuses.set(update.id, update); this.messenger?.send("indexing/statusUpdate", update); } + // A way for gui to retrieve initial statuses + async initStatuses(): Promise<void> { + if (!this.config?.docs) { + return; + } + const metadata = await this.listMetadata(); + + this.config.docs?.forEach((doc) => { + if (!doc.startUrl) { + console.error("Invalid config docs entry", doc); + return; + } + + const sharedStatus = { + type: "docs" as IndexingStatus["type"], + id: doc.startUrl, + embeddingsProviderId: this.config.embeddingsProvider.id, + isReindexing: false, + title: doc.title, + debugInfo: `max depth: ${doc.maxDepth}`, + icon: doc.faviconUrl, + url: doc.startUrl, + }; + const indexedStatus: IndexingStatus = metadata.find( + (meta) => meta.startUrl === doc.startUrl, + ) + ? { + ...sharedStatus, + progress: 0, + description: "Pending", + status: "pending", + } + : { + ...sharedStatus, + progress: 1, + description: "Complete", + status: "complete", + }; + this.handleStatusUpdate(indexedStatus); + }); + } + abort(startUrl: string) { const status = this.statuses.get(startUrl); if (status) { @@ -362,6 +377,7 @@ export default class DocsService { const indexExists = await this.hasMetadata(startUrl); + // Build status update - most of it is fixed values const fixedStatus: Pick< IndexingStatus, | "type" @@ -383,6 +399,8 @@ export default class DocsService { url: siteIndexingConfig.startUrl, }; + // Clear current indexes if reIndexing + // if (indexExists) { if (reIndex) { await this.deleteIndexes(startUrl); @@ -398,6 +416,12 @@ export default class DocsService { } } + // If not preindexed + const isPreIndexedDoc = !!preIndexedDocs[siteIndexingConfig.startUrl]; + if (!isPreIndexedDoc) { + this.addToConfig(siteIndexingConfig); + } + try { this.handleStatusUpdate({ ...fixedStatus, @@ -435,7 +459,7 @@ export default class DocsService { articles.push(article); processedPages++; - await new Promise((resolve) => setTimeout(resolve, 50)); // Locks down GUI if no sleeping + await new Promise((resolve) => setTimeout(resolve, 100)); // Locks down GUI if no sleeping } void Telemetry.capture("docs_pages_crawled", { @@ -788,19 +812,13 @@ export default class DocsService { } } - // Sends "Pending" or current status for each - await this.initStatuses(); - - for (const doc of changedDocs) { - console.log(`Updating indexed doc: ${doc.startUrl}`); - await this.indexAndAdd(doc, true); - } + await Promise.allSettled([ + ...changedDocs.map((doc) => this.indexAndAdd(doc, true)), + ...newDocs.map((doc) => this.indexAndAdd(doc)), + ]); for (const doc of newDocs) { - console.log(`Indexing new doc: ${doc.startUrl}`); void Telemetry.capture("add_docs_config", { url: doc.startUrl }); - - await this.indexAndAdd(doc); } for (const doc of deletedDocs) { @@ -941,7 +959,7 @@ export default class DocsService { ); } - private addToConfig({ siteIndexingConfig }: AddParams) { + private addToConfig(siteIndexingConfig: SiteIndexingConfig) { // Handles the case where a user has manually added the doc to config.json // so it already exists in the file const doesDocExist = this.config.docs?.some( @@ -959,13 +977,6 @@ export default class DocsService { private async add(params: AddParams) { await this.addToLance(params); await this.addMetadataToSqlite(params); - - const isPreIndexedDoc = - !!preIndexedDocs[params.siteIndexingConfig.startUrl]; - - if (!isPreIndexedDoc) { - this.addToConfig(params); - } } // Delete methods @@ -1030,10 +1041,7 @@ export default class DocsService { console.log( `Reindexing non-preindexed docs with new embeddings provider: ${embeddingsProvider.id}`, ); - await this.initStatuses(); - for (const doc of docs) { - await this.indexAndAdd(doc, true); - } + await Promise.allSettled(docs.map((doc) => this.indexAndAdd(doc))); // Important that this only is invoked after we have successfully // cleared and reindex the docs so that the table cannot end up in an diff --git a/core/protocol/core.ts b/core/protocol/core.ts index f5b04c9cf5..88cb5642c9 100644 --- a/core/protocol/core.ts +++ b/core/protocol/core.ts @@ -9,6 +9,7 @@ import type { DiffLine, FileSymbolMap, IdeSettings, + IndexingStatus, LLMFullCompletionOptions, MessageContent, ModelDescription, @@ -165,8 +166,8 @@ export type ToCoreFromIdeOrWebviewProtocol = { "indexing/reindex": [{ type: string; id: string }, void]; "indexing/abort": [{ type: string; id: string }, void]; "indexing/setPaused": [{ type: string; id: string; paused: boolean }, void]; - "indexing/initStatuses": [undefined, void]; "docs/getSuggestedDocs": [undefined, void]; + "docs/initStatuses": [undefined, void]; addAutocompleteModel: [{ model: ModelDescription }, void]; diff --git a/core/protocol/passThrough.ts b/core/protocol/passThrough.ts index 2bd8de2c18..720a6f66e8 100644 --- a/core/protocol/passThrough.ts +++ b/core/protocol/passThrough.ts @@ -48,11 +48,11 @@ export const WEBVIEW_TO_CORE_PASS_THROUGH: (keyof ToCoreFromWebviewProtocol)[] = "index/forceReIndexFiles", "index/indexingProgressBarInitialized", // Docs, etc. - "indexing/initStatuses", "indexing/reindex", "indexing/abort", "indexing/setPaused", "docs/getSuggestedDocs", + "docs/initStatuses", // "completeOnboarding", "addAutocompleteModel", diff --git a/gui/src/App.tsx b/gui/src/App.tsx index 1e34e6e156..ca807a8764 100644 --- a/gui/src/App.tsx +++ b/gui/src/App.tsx @@ -8,7 +8,7 @@ import useSubmenuContextProviders from "./hooks/useSubmenuContextProviders"; import { useVscTheme } from "./hooks/useVscTheme"; import { AddNewModel, ConfigureProvider } from "./pages/AddNewModel"; import ConfigErrorPage from "./pages/config-error"; -import Edit from "./pages/Edit"; +import Edit from "./pages/edit"; import ErrorPage from "./pages/error"; import Chat from "./pages/gui"; import History from "./pages/history"; diff --git a/gui/src/components/AccountDialog.tsx b/gui/src/components/AccountDialog.tsx index db61fe3c0e..5438d11df6 100644 --- a/gui/src/components/AccountDialog.tsx +++ b/gui/src/components/AccountDialog.tsx @@ -65,6 +65,8 @@ function AccountDialog() { ); } + const dispatch = useDispatch(); + const changeProfileId = (id: string) => { ideMessenger.post("didChangeSelectedProfile", { id }); dispatch(setSelectedProfileId(id)); diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx index 68039e85c3..b781a5ec8d 100644 --- a/gui/src/components/dialogs/AddDocsDialog.tsx +++ b/gui/src/components/dialogs/AddDocsDialog.tsx @@ -22,7 +22,7 @@ import { setShowDialog, } from "../../redux/slices/uiStateSlice"; import { RootState } from "../../redux/store"; -import IndexingStatusViewer from "../indexing/IndexingStatus"; +import IndexingStatusViewer from "../indexing/DocsIndexingStatus"; import { ToolTip } from "../gui/Tooltip"; import FileIcon from "../FileIcon"; diff --git a/gui/src/components/indexing/DocsIndexingPeeks.tsx b/gui/src/components/indexing/DocsIndexingPeeks.tsx index e50b4adcf2..d63012865c 100644 --- a/gui/src/components/indexing/DocsIndexingPeeks.tsx +++ b/gui/src/components/indexing/DocsIndexingPeeks.tsx @@ -1,4 +1,4 @@ -import { useDispatch } from "react-redux"; +import { useDispatch, useSelector } from "react-redux"; import { IndexingStatus } from "core"; import { useMemo } from "react"; @@ -8,6 +8,7 @@ import { setDialogMessage, setShowDialog, } from "../../redux/slices/uiStateSlice"; +import { RootState } from "../../redux/store"; export interface DocsIndexingPeekProps { status: IndexingStatus; @@ -58,6 +59,9 @@ interface DocsIndexingPeeksProps { } function DocsIndexingPeekList({ statuses }: DocsIndexingPeeksProps) { + const config = useSelector((store: RootState) => store.state.config); + const configDocs = config.docs ?? []; + // const docs return ( <div className="border-vsc-input-border mt-2 flex flex-col border-0 border-t border-solid pt-2"> <p className="mx-0 my-1.5 p-0 px-1 font-semibold text-stone-500"> diff --git a/gui/src/components/indexing/IndexingStatus.tsx b/gui/src/components/indexing/DocsIndexingStatus.tsx similarity index 68% rename from gui/src/components/indexing/IndexingStatus.tsx rename to gui/src/components/indexing/DocsIndexingStatus.tsx index 6fe72c7419..4fb5013cd4 100644 --- a/gui/src/components/indexing/IndexingStatus.tsx +++ b/gui/src/components/indexing/DocsIndexingStatus.tsx @@ -1,6 +1,6 @@ -import { IndexingStatus } from "core"; +import { IndexingStatus, SiteIndexingConfig } from "core"; import { PropsWithChildren, useContext, useMemo } from "react"; -import { useDispatch } from "react-redux"; +import { useDispatch, useSelector } from "react-redux"; import { usePostHog } from "posthog-js/react"; import { IdeMessengerContext } from "../../context/IdeMessenger"; import { @@ -12,9 +12,10 @@ import { XMarkIcon, } from "@heroicons/react/24/outline"; import { updateIndexingStatus } from "../../redux/slices/stateSlice"; +import { RootState } from "../../redux/store"; interface IndexingStatusViewerProps { - status: IndexingStatus; + docConfig: SiteIndexingConfig; } const STATUS_TO_ICON: Record<IndexingStatus["status"], any> = { @@ -27,33 +28,43 @@ const STATUS_TO_ICON: Record<IndexingStatus["status"], any> = { failed: XMarkIcon, // Since we show an erorr message below }; -function IndexingStatusViewer({ status }: IndexingStatusViewerProps) { +function DocsIndexingStatus({ docConfig }: IndexingStatusViewerProps) { const ideMessenger = useContext(IdeMessengerContext); const posthog = usePostHog(); const dispatch = useDispatch(); + // const status = undefined; + const status = useSelector( + (store: RootState) => store.state.indexing.statuses[docConfig.startUrl], + ); + const reIndex = () => ideMessenger.post("indexing/reindex", { - type: status.type, - id: status.id, + type: "docs", + id: docConfig.startUrl, }); const abort = () => { ideMessenger.post("indexing/abort", { - type: status.type, - id: status.id, + type: "docs", + id: docConfig.startUrl, }); // Optimistic abort status - dispatch( - updateIndexingStatus({ ...status, status: "aborted", progress: 0 }), - ); + if (status) { + dispatch( + updateIndexingStatus({ ...status, status: "aborted", progress: 0 }), + ); + } }; const progressPercentage = useMemo(() => { + if (!status) { + return 0; + } return Math.min(100, Math.max(0, status.progress * 100)); - }, [status.progress]); + }, [status?.progress]); - const Icon = STATUS_TO_ICON[status.status]; + const Icon = STATUS_TO_ICON[status?.status]; return ( <div className="mt-2 flex w-full flex-col"> @@ -62,24 +73,26 @@ function IndexingStatusViewer({ status }: IndexingStatusViewerProps) { className={`flex flex-row items-center justify-between gap-2 text-sm`} > <div - className={`flex flex-row items-center gap-2 ${status.url ? "cursor-pointer hover:underline" : ""}`} + className={`flex flex-row items-center gap-2 ${status?.url ? "cursor-pointer hover:underline" : ""}`} onClick={() => { - if (status.url) { + if (status?.url) { ideMessenger.post("openUrl", status.url); } }} > - {status.icon ? ( - <img src={status.icon} alt="doc icon" className="h-4 w-4" /> + {docConfig.faviconUrl ? ( + <img + src={docConfig.faviconUrl} + alt="doc icon" + className="h-4 w-4" + /> ) : null} <p className="lines lines-1 m-0 p-0 text-left"> - {status.title ?? status.id} + {docConfig.title ?? docConfig.startUrl} </p> - {!!status.url && ( - <ArrowTopRightOnSquareIcon className="mb-0.5 h-3 w-3 text-stone-500" /> - )} + <ArrowTopRightOnSquareIcon className="mb-0.5 h-3 w-3 text-stone-500" /> </div> - {status.status === "pending" ? ( + {status?.status === "pending" ? ( <div className="text-xs text-stone-500">Pending...</div> ) : ( <div className="flex flex-row items-center gap-1 text-stone-500"> @@ -87,7 +100,7 @@ function IndexingStatusViewer({ status }: IndexingStatusViewerProps) { {Icon ? ( <Icon className={`inline-block h-4 w-4 text-stone-500 ${ - status.status === "indexing" ? "animate-spin-slow" : "" + status?.status === "indexing" ? "animate-spin-slow" : "" }`} ></Icon> ) : null} @@ -98,7 +111,7 @@ function IndexingStatusViewer({ status }: IndexingStatusViewerProps) { <div className="my-2 h-1.5 w-full rounded-md border border-solid border-gray-400"> <div className={`h-full rounded-lg transition-all duration-200 ease-in-out ${ - status.status === "failed" ? "bg-red-600" : "bg-stone-500" + status?.status === "failed" ? "bg-red-600" : "bg-stone-500" }`} style={{ width: `${progressPercentage}%`, @@ -118,7 +131,7 @@ function IndexingStatusViewer({ status }: IndexingStatusViewerProps) { paused: () => {}, deleted: () => {}, pending: () => {}, - }[status.status] + }[status?.status] } > { @@ -130,18 +143,16 @@ function IndexingStatusViewer({ status }: IndexingStatusViewerProps) { paused: "", deleted: "", pending: "", - }[status.status] + }[status?.status] } </span> - {/* {status.status === "indexing" && ( */} <span className="lines lines-1 text-right text-xs text-stone-500"> - {status.description} + {status?.description} </span> - {/* )} */} </div> </div> ); } -export default IndexingStatusViewer; +export default DocsIndexingStatus; diff --git a/gui/src/components/indexing/DocsIndexingStatuses.tsx b/gui/src/components/indexing/DocsIndexingStatuses.tsx new file mode 100644 index 0000000000..84fdba9031 --- /dev/null +++ b/gui/src/components/indexing/DocsIndexingStatuses.tsx @@ -0,0 +1,57 @@ +import { useMemo } from "react"; +import { useDispatch, useSelector } from "react-redux"; +import { RootState } from "../../redux/store"; +import IndexingStatusViewer from "./DocsIndexingStatus"; +import { SecondaryButton } from ".."; +import { + setDialogMessage, + setShowDialog, +} from "../../redux/slices/uiStateSlice"; +import AddDocsDialog from "../dialogs/AddDocsDialog"; +import DocsIndexingStatus from "./DocsIndexingStatus"; + +function DocsIndexingStatuses() { + const dispatch = useDispatch(); + const config = useSelector((store: RootState) => store.state.config); + const configDocs = config.docs ?? []; + + // const indexingStatuses = useSelector( + // (store: RootState) => store.state.indexing.statuses, + // ); + // const docsStatuses = useMemo(() => { + // const docs = Object.values(indexingStatuses).filter( + // (status) => status.type === "docs" && status.status !== "deleted", + // ); + // return docs; + // }, [indexingStatuses]); + + return ( + <div className="flex flex-col gap-1"> + <div className="flex flex-row items-center justify-between"> + <h3 className="mb-1 mt-0 text-xl">@docs indexes</h3> + {configDocs.length ? ( + <SecondaryButton + className="border-vsc-foreground text-vsc-foreground enabled:hover:bg-vsc-background m-2 rounded border bg-inherit px-3 py-2 enabled:hover:cursor-pointer enabled:hover:opacity-90 disabled:text-gray-500" + type="submit" + onClick={() => { + dispatch(setShowDialog(true)); + dispatch(setDialogMessage(<AddDocsDialog />)); + }} + > + Add + </SecondaryButton> + ) : null} + </div> + <span className="text-xs text-stone-500"> + Manage your documentation sources + </span> + <div className="flex max-h-[170px] flex-col gap-1 overflow-x-hidden overflow-y-scroll pr-2"> + {configDocs.map((doc) => { + return <DocsIndexingStatus key={doc.startUrl} docConfig={doc} />; + })} + </div> + </div> + ); +} + +export default DocsIndexingStatuses; diff --git a/gui/src/components/indexing/IndexingStatuses.tsx b/gui/src/components/indexing/IndexingStatuses.tsx deleted file mode 100644 index 3ea04216f4..0000000000 --- a/gui/src/components/indexing/IndexingStatuses.tsx +++ /dev/null @@ -1,71 +0,0 @@ -import { useMemo } from "react"; -import { useDispatch, useSelector } from "react-redux"; -import { RootState } from "../../redux/store"; -import IndexingStatusViewer from "./IndexingStatus"; -import { Button, SecondaryButton } from ".."; -import { - setDialogMessage, - setShowDialog, -} from "../../redux/slices/uiStateSlice"; -import AddDocsDialog from "../dialogs/AddDocsDialog"; -import { PlusCircleIcon } from "@heroicons/react/24/outline"; -import { ToolTip } from "../gui/Tooltip"; - -function IndexingStatuses() { - const indexingStatuses = useSelector( - (store: RootState) => store.state.indexing.statuses, - ); - const docsStatuses = useMemo(() => { - const docs = Object.values(indexingStatuses).filter( - (status) => status.type === "docs" && status.status !== "deleted", - ); - return docs; - }, [indexingStatuses]); - const dispatch = useDispatch(); - - return ( - <div className="flex flex-col gap-1"> - <div className="flex flex-row items-center justify-between"> - <h3 className="mb-1 mt-0 text-xl">@docs indexes</h3> - {docsStatuses.length ? ( - <div - className="border-vsc-foreground text-vsc-foreground enabled:hover:bg-vsc-background m-2 rounded border border-solid bg-inherit px-3 py-2 enabled:hover:cursor-pointer enabled:hover:opacity-90 disabled:text-gray-500" - onClick={() => { - dispatch(setShowDialog(true)); - dispatch(setDialogMessage(<AddDocsDialog />)); - }} - > - Add - </div> - ) : // <div> - // <PlusCircleIcon - // data-tooltip-id={"more-add-docs-button"} - // className="text-vsc-foreground-muted h-5 w-5 cursor-pointer focus:ring-0" - // onClick={() => { - // dispatch(setShowDialog(true)); - // dispatch(setDialogMessage(<AddDocsDialog />)); - // }} - // /> - // <ToolTip id={"more-add-docs-button"} place="top"> - // Add Docs - // </ToolTip> - // </div> - null} - </div> - <span className="text-xs text-stone-500"> - Manage your documentation sources - </span> - <div className="flex max-h-[170px] flex-col gap-1 overflow-x-hidden overflow-y-scroll pr-2"> - {docsStatuses.length ? ( - docsStatuses.map((status) => { - return <IndexingStatusViewer key={status.id} status={status} />; - }) - ) : ( - <SecondaryButton>Add Docs</SecondaryButton> - )} - </div> - </div> - ); -} - -export default IndexingStatuses; diff --git a/gui/src/hooks/useSetup.ts b/gui/src/hooks/useSetup.ts index ecc7a7ac92..34a1e2f748 100644 --- a/gui/src/hooks/useSetup.ts +++ b/gui/src/hooks/useSetup.ts @@ -54,9 +54,12 @@ function useSetup(dispatch: Dispatch) { loadConfig(); const interval = setInterval(() => { if (initialConfigLoad.current) { + // Docs init on config load + ideMessenger.post("docs/getSuggestedDocs", undefined); + ideMessenger.post("docs/initStatuses", undefined); + // This triggers sending pending status to the GUI for relevant docs indexes clearInterval(interval); - ideMessenger.post("indexing/initStatuses", undefined); return; } loadConfig(); @@ -85,8 +88,6 @@ function useSetup(dispatch: Dispatch) { // ON LOAD useEffect(() => { - ideMessenger.post("docs/getSuggestedDocs", undefined); - // Override persisted state dispatch(setInactive()); diff --git a/gui/src/pages/More/More.tsx b/gui/src/pages/More/More.tsx index e743a38343..81648b41b6 100644 --- a/gui/src/pages/More/More.tsx +++ b/gui/src/pages/More/More.tsx @@ -15,7 +15,7 @@ import { setOnboardingCard } from "../../redux/slices/uiStateSlice"; import useHistory from "../../hooks/useHistory"; import MoreHelpRow from "./MoreHelpRow"; import IndexingProgress from "./IndexingProgress"; -import IndexingStatuses from "../../components/indexing/IndexingStatuses"; +import DocsIndexingStatuses from "../../components/indexing/DocsIndexingStatuses"; function MorePage() { useNavigationListener(); @@ -52,7 +52,7 @@ function MorePage() { <IndexingProgress /> </div> <div className="flex flex-col py-5"> - <IndexingStatuses /> + <DocsIndexingStatuses /> </div> <div className="py-5"> diff --git a/gui/src/redux/slices/stateSlice.ts b/gui/src/redux/slices/stateSlice.ts index 42fea2a60b..65075d87cd 100644 --- a/gui/src/redux/slices/stateSlice.ts +++ b/gui/src/redux/slices/stateSlice.ts @@ -45,7 +45,7 @@ type State = { nextCodeBlockToApplyIndex: number; indexing: { hiddenChatPeekTypes: Record<IndexingStatus["type"], boolean>; - statuses: Record<string, IndexingStatus>; + statuses: Record<string, IndexingStatus>; // status id -> status }; streamAborter: AbortController; docsSuggestions: PackageDocsResult[]; diff --git a/manual-testing-sandbox/package.json b/manual-testing-sandbox/nested-folder/package.json similarity index 100% rename from manual-testing-sandbox/package.json rename to manual-testing-sandbox/nested-folder/package.json From a541fe2f381a5ca3154a22e9ec885841c0d7ca2e Mon Sep 17 00:00:00 2001 From: Dallin Romney <dallinromney@gmail.com> Date: Fri, 22 Nov 2024 16:03:44 -0800 Subject: [PATCH 12/17] docs suggestions v2 --- gui/src/components/dialogs/AddDocsDialog.tsx | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx index b781a5ec8d..00134a6f73 100644 --- a/gui/src/components/dialogs/AddDocsDialog.tsx +++ b/gui/src/components/dialogs/AddDocsDialog.tsx @@ -1,35 +1,25 @@ import { - CheckCircleIcon, CheckIcon, - ChevronDownIcon, - ChevronUpIcon, - CodeBracketIcon, - ExclamationTriangleIcon, InformationCircleIcon, - LinkIcon, PencilIcon, - PlusCircleIcon, PlusIcon, } from "@heroicons/react/24/outline"; import { IndexingStatus, PackageDocsResult, SiteIndexingConfig } from "core"; import { usePostHog } from "posthog-js/react"; import { useContext, useLayoutEffect, useMemo, useRef, useState } from "react"; import { useDispatch, useSelector } from "react-redux"; -import { Button, HelperText, Input, lightGray, SecondaryButton } from ".."; +import { Input, SecondaryButton } from ".."; import { IdeMessengerContext } from "../../context/IdeMessenger"; import { setDialogMessage, setShowDialog, } from "../../redux/slices/uiStateSlice"; import { RootState } from "../../redux/store"; -import IndexingStatusViewer from "../indexing/DocsIndexingStatus"; - import { ToolTip } from "../gui/Tooltip"; import FileIcon from "../FileIcon"; import DocsIndexingPeeks from "../indexing/DocsIndexingPeeks"; import { updateIndexingStatus } from "../../redux/slices/stateSlice"; import preIndexedDocs from "core/indexing/docs/preIndexedDocs"; -import { NewSessionButton } from "../mainInput/NewSessionButton"; function AddDocsDialog() { const posthog = usePostHog(); @@ -99,6 +89,7 @@ function AddDocsDialog() { dispatch(setShowDialog(false)); dispatch(setDialogMessage(undefined)); }; + function onSubmit(e: any) { e.preventDefault(); From de984454cd4978cce51ce13a5539a0984b6162b7 Mon Sep 17 00:00:00 2001 From: Dallin Romney <dallinromney@gmail.com> Date: Mon, 25 Nov 2024 12:25:20 -0800 Subject: [PATCH 13/17] status race condition fix --- core/indexing/docs/DocsService.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/indexing/docs/DocsService.ts b/core/indexing/docs/DocsService.ts index f9c88cd0ed..0f3724d61d 100644 --- a/core/indexing/docs/DocsService.ts +++ b/core/indexing/docs/DocsService.ts @@ -150,10 +150,15 @@ export default class DocsService { this.config.docs?.forEach((doc) => { if (!doc.startUrl) { - console.error("Invalid config docs entry", doc); + console.error("Invalid config docs entry, no start"); return; } + const currentStatus = this.statuses.get(doc.startUrl); + if (currentStatus) { + return currentStatus; + } + const sharedStatus = { type: "docs" as IndexingStatus["type"], id: doc.startUrl, From d26f82a3fe11c2aac005f4a8deaa81b2cdc37b74 Mon Sep 17 00:00:00 2001 From: Dallin Romney <dallinromney@gmail.com> Date: Mon, 25 Nov 2024 16:47:54 -0800 Subject: [PATCH 14/17] edit casing bug --- gui/src/App.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gui/src/App.tsx b/gui/src/App.tsx index ca807a8764..82433e2052 100644 --- a/gui/src/App.tsx +++ b/gui/src/App.tsx @@ -8,7 +8,7 @@ import useSubmenuContextProviders from "./hooks/useSubmenuContextProviders"; import { useVscTheme } from "./hooks/useVscTheme"; import { AddNewModel, ConfigureProvider } from "./pages/AddNewModel"; import ConfigErrorPage from "./pages/config-error"; -import Edit from "./pages/edit"; +import Edit from "./pages/edit/Edit"; import ErrorPage from "./pages/error"; import Chat from "./pages/gui"; import History from "./pages/history"; From 800e65e51ccc0c978a44bc10de0dedb7b35516dd Mon Sep 17 00:00:00 2001 From: Dallin Romney <dallinromney@gmail.com> Date: Mon, 25 Nov 2024 17:01:16 -0800 Subject: [PATCH 15/17] tiny tiny --- gui/src/App.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gui/src/App.tsx b/gui/src/App.tsx index 82433e2052..ca807a8764 100644 --- a/gui/src/App.tsx +++ b/gui/src/App.tsx @@ -8,7 +8,7 @@ import useSubmenuContextProviders from "./hooks/useSubmenuContextProviders"; import { useVscTheme } from "./hooks/useVscTheme"; import { AddNewModel, ConfigureProvider } from "./pages/AddNewModel"; import ConfigErrorPage from "./pages/config-error"; -import Edit from "./pages/edit/Edit"; +import Edit from "./pages/edit"; import ErrorPage from "./pages/error"; import Chat from "./pages/gui"; import History from "./pages/history"; From 7f5ec0dfc10f9151c8292e5f53d29692ebb7b0df Mon Sep 17 00:00:00 2001 From: Dallin Romney <dallinromney@gmail.com> Date: Mon, 25 Nov 2024 17:36:16 -0800 Subject: [PATCH 16/17] short --- gui/src/pages/{Edit => edi}/AddFileButton.tsx | 0 gui/src/pages/{Edit => edi}/CodeToEdit.tsx | 0 gui/src/pages/{Edit => edi}/CodeToEditListItem.tsx | 0 gui/src/pages/{Edit => edi}/Edit.tsx | 0 gui/src/pages/{Edit => edi}/getMultifileEditPrompt.ts | 0 gui/src/pages/{Edit => edi}/index.tsx | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename gui/src/pages/{Edit => edi}/AddFileButton.tsx (100%) rename gui/src/pages/{Edit => edi}/CodeToEdit.tsx (100%) rename gui/src/pages/{Edit => edi}/CodeToEditListItem.tsx (100%) rename gui/src/pages/{Edit => edi}/Edit.tsx (100%) rename gui/src/pages/{Edit => edi}/getMultifileEditPrompt.ts (100%) rename gui/src/pages/{Edit => edi}/index.tsx (100%) diff --git a/gui/src/pages/Edit/AddFileButton.tsx b/gui/src/pages/edi/AddFileButton.tsx similarity index 100% rename from gui/src/pages/Edit/AddFileButton.tsx rename to gui/src/pages/edi/AddFileButton.tsx diff --git a/gui/src/pages/Edit/CodeToEdit.tsx b/gui/src/pages/edi/CodeToEdit.tsx similarity index 100% rename from gui/src/pages/Edit/CodeToEdit.tsx rename to gui/src/pages/edi/CodeToEdit.tsx diff --git a/gui/src/pages/Edit/CodeToEditListItem.tsx b/gui/src/pages/edi/CodeToEditListItem.tsx similarity index 100% rename from gui/src/pages/Edit/CodeToEditListItem.tsx rename to gui/src/pages/edi/CodeToEditListItem.tsx diff --git a/gui/src/pages/Edit/Edit.tsx b/gui/src/pages/edi/Edit.tsx similarity index 100% rename from gui/src/pages/Edit/Edit.tsx rename to gui/src/pages/edi/Edit.tsx diff --git a/gui/src/pages/Edit/getMultifileEditPrompt.ts b/gui/src/pages/edi/getMultifileEditPrompt.ts similarity index 100% rename from gui/src/pages/Edit/getMultifileEditPrompt.ts rename to gui/src/pages/edi/getMultifileEditPrompt.ts diff --git a/gui/src/pages/Edit/index.tsx b/gui/src/pages/edi/index.tsx similarity index 100% rename from gui/src/pages/Edit/index.tsx rename to gui/src/pages/edi/index.tsx From f7ce697b3f27d2cf88dc67987c94b1fba75b2fb5 Mon Sep 17 00:00:00 2001 From: Dallin Romney <dallinromney@gmail.com> Date: Mon, 25 Nov 2024 17:36:32 -0800 Subject: [PATCH 17/17] long --- gui/src/pages/{edi => edit}/AddFileButton.tsx | 0 gui/src/pages/{edi => edit}/CodeToEdit.tsx | 0 gui/src/pages/{edi => edit}/CodeToEditListItem.tsx | 0 gui/src/pages/{edi => edit}/Edit.tsx | 0 gui/src/pages/{edi => edit}/getMultifileEditPrompt.ts | 0 gui/src/pages/{edi => edit}/index.tsx | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename gui/src/pages/{edi => edit}/AddFileButton.tsx (100%) rename gui/src/pages/{edi => edit}/CodeToEdit.tsx (100%) rename gui/src/pages/{edi => edit}/CodeToEditListItem.tsx (100%) rename gui/src/pages/{edi => edit}/Edit.tsx (100%) rename gui/src/pages/{edi => edit}/getMultifileEditPrompt.ts (100%) rename gui/src/pages/{edi => edit}/index.tsx (100%) diff --git a/gui/src/pages/edi/AddFileButton.tsx b/gui/src/pages/edit/AddFileButton.tsx similarity index 100% rename from gui/src/pages/edi/AddFileButton.tsx rename to gui/src/pages/edit/AddFileButton.tsx diff --git a/gui/src/pages/edi/CodeToEdit.tsx b/gui/src/pages/edit/CodeToEdit.tsx similarity index 100% rename from gui/src/pages/edi/CodeToEdit.tsx rename to gui/src/pages/edit/CodeToEdit.tsx diff --git a/gui/src/pages/edi/CodeToEditListItem.tsx b/gui/src/pages/edit/CodeToEditListItem.tsx similarity index 100% rename from gui/src/pages/edi/CodeToEditListItem.tsx rename to gui/src/pages/edit/CodeToEditListItem.tsx diff --git a/gui/src/pages/edi/Edit.tsx b/gui/src/pages/edit/Edit.tsx similarity index 100% rename from gui/src/pages/edi/Edit.tsx rename to gui/src/pages/edit/Edit.tsx diff --git a/gui/src/pages/edi/getMultifileEditPrompt.ts b/gui/src/pages/edit/getMultifileEditPrompt.ts similarity index 100% rename from gui/src/pages/edi/getMultifileEditPrompt.ts rename to gui/src/pages/edit/getMultifileEditPrompt.ts diff --git a/gui/src/pages/edi/index.tsx b/gui/src/pages/edit/index.tsx similarity index 100% rename from gui/src/pages/edi/index.tsx rename to gui/src/pages/edit/index.tsx