diff --git a/core/config/ProfileLifecycleManager.ts b/core/config/ProfileLifecycleManager.ts index 85116f2133..8d44c8bac9 100644 --- a/core/config/ProfileLifecycleManager.ts +++ b/core/config/ProfileLifecycleManager.ts @@ -1,5 +1,3 @@ -import { config } from "dotenv"; - import { BrowserSerializedContinueConfig, ContinueConfig, diff --git a/core/config/load.ts b/core/config/load.ts index b692b36bf2..9911c840fe 100644 --- a/core/config/load.ts +++ b/core/config/load.ts @@ -527,6 +527,7 @@ function finalToBrowserConfig( embeddingsProvider: final.embeddingsProvider?.id, ui: final.ui, experimental: final.experimental, + docs: final.docs, }; } diff --git a/core/core.ts b/core/core.ts index 6c8d20537a..f5453ea6d3 100644 --- a/core/core.ts +++ b/core/core.ts @@ -19,6 +19,7 @@ import { ControlPlaneClient } from "./control-plane/client"; import { streamDiffLines } from "./edit/streamDiffLines"; import { CodebaseIndexer, PauseToken } from "./indexing/CodebaseIndexer"; import DocsService from "./indexing/docs/DocsService"; +import { getAllSuggestedDocs } from "./indexing/docs/suggestions"; import { defaultIgnoreFile } from "./indexing/ignore.js"; import Ollama from "./llm/llms/Ollama"; import { createNewPromptFileV2 } from "./promptFiles/v2/createNewPromptFile"; @@ -28,11 +29,7 @@ import { DevDataSqliteDb } from "./util/devdataSqlite"; import { fetchwithRequestOptions } from "./util/fetchWithOptions"; import { GlobalContext } from "./util/GlobalContext"; import historyManager from "./util/history"; -import { - editConfigJson, - getConfigJsonPath, - setupInitialDotContinueDirectory, -} from "./util/paths"; +import { editConfigJson, setupInitialDotContinueDirectory } from "./util/paths"; import { Telemetry } from "./util/posthog"; import { getSymbolsForManyFiles } from "./util/treeSitter"; import { TTS } from "./util/tts"; @@ -734,8 +731,16 @@ export class Core { this.docsService.setPaused(msg.data.id, msg.data.paused); } }); - on("indexing/initStatuses", async (msg) => { - return this.docsService.initStatuses(); + on("docs/getSuggestedDocs", async (msg) => { + if (hasRequestedDocs) { + return; + } // TODO, remove, hack because of rerendering + hasRequestedDocs = true; + const suggestedDocs = await getAllSuggestedDocs(this.ide); + this.messenger.send("docs/suggestions", suggestedDocs); + }); + on("docs/initStatuses", async (msg) => { + void this.docsService.initStatuses(); }); // @@ -841,3 +846,5 @@ export class Core { // private } + +let hasRequestedDocs = false; diff --git a/core/index.d.ts b/core/index.d.ts index 55c1e8ac79..a64cb88ee2 100644 --- a/core/index.d.ts +++ b/core/index.d.ts @@ -68,8 +68,6 @@ export interface IndexingStatus { url?: string; } -export type IndexingStatusMap = Map; - export type PromptTemplateFunction = ( history: ChatMessage[], otherData: Record, @@ -1213,4 +1211,42 @@ export interface BrowserSerializedContinueConfig { reranker?: RerankerDescription; experimental?: ExperimentalConfig; analytics?: AnalyticsConfig; + docs?: SiteIndexingConfig[]; +} + +// DOCS SUGGESTIONS AND PACKAGE INFO +export interface FilePathAndName { + path: string; + name: string; } + +export interface PackageFilePathAndName extends FilePathAndName { + packageRegistry: string; // e.g. npm, pypi +} + +export type ParsedPackageInfo = { + name: string; + packageFile: PackageFilePathAndName; + language: string; + version: string; +}; + +export type PackageDetails = { + docsLink?: string; + docsLinkWarning?: string; + title?: string; + description?: string; + repo?: string; + license?: string; +}; + +export type PackageDetailsSuccess = PackageDetails & { + docsLink: string; +}; + +export type PackageDocsResult = { + packageInfo: ParsedPackageInfo; +} & ( + | { error: string; details?: never } + | { details: PackageDetailsSuccess; error?: never } +); diff --git a/core/indexing/docs/DocsService.ts b/core/indexing/docs/DocsService.ts index eb639eea51..6e841f6be2 100644 --- a/core/indexing/docs/DocsService.ts +++ b/core/indexing/docs/DocsService.ts @@ -7,7 +7,6 @@ import { ContinueConfig, EmbeddingsProvider, IDE, - IndexingStatusMap, IndexingStatus, SiteIndexingConfig, IdeInfo, @@ -87,7 +86,6 @@ export type AddParams = { export default class DocsService { static lanceTableName = "docs"; static sqlitebTableName = "docs"; - static indexingType = "docs"; static preIndexedDocsEmbeddingsProvider = new TransformersJsEmbeddingsProvider(); @@ -137,36 +135,58 @@ export default class DocsService { configHandler.onConfigUpdate(this.handleConfigUpdate.bind(this)); } - readonly statuses: IndexingStatusMap = new Map(); + readonly statuses: Map = new Map(); + + handleStatusUpdate(update: IndexingStatus) { + this.statuses.set(update.id, update); + this.messenger?.send("indexing/statusUpdate", update); + } + + // A way for gui to retrieve initial statuses + async initStatuses(): Promise { + if (!this.config?.docs) { + return; + } + const metadata = await this.listMetadata(); + + this.config.docs?.forEach((doc) => { + if (!doc.startUrl) { + console.error("Invalid config docs entry, no start"); + return; + } - // Function for GUI to retrieve initial pending statuses - // And kickoff indexing where needed - async initStatuses() { - this.config?.docs?.forEach(async (doc) => { const currentStatus = this.statuses.get(doc.startUrl); if (currentStatus) { - this.handleStatusUpdate(currentStatus); - } else { - this.handleStatusUpdate({ - type: "docs", - id: doc.startUrl, - embeddingsProviderId: this.config.embeddingsProvider.id, - isReindexing: false, - progress: 0, - description: "Pending", - status: "pending", - title: doc.title, - debugInfo: `max depth: ${doc.maxDepth}`, - icon: doc.faviconUrl, - url: doc.startUrl, - }); + return currentStatus; } - }); - } - handleStatusUpdate(update: IndexingStatus) { - this.statuses.set(update.id, update); - this.messenger?.send("indexing/statusUpdate", update); + const sharedStatus = { + type: "docs" as IndexingStatus["type"], + id: doc.startUrl, + embeddingsProviderId: this.config.embeddingsProvider.id, + isReindexing: false, + title: doc.title, + debugInfo: `max depth: ${doc.maxDepth}`, + icon: doc.faviconUrl, + url: doc.startUrl, + }; + const indexedStatus: IndexingStatus = metadata.find( + (meta) => meta.startUrl === doc.startUrl, + ) + ? { + ...sharedStatus, + progress: 0, + description: "Pending", + status: "pending", + } + : { + ...sharedStatus, + progress: 1, + description: "Complete", + status: "complete", + }; + this.handleStatusUpdate(indexedStatus); + }); } abort(startUrl: string) { @@ -363,6 +383,7 @@ export default class DocsService { const indexExists = await this.hasMetadata(startUrl); + // Build status update - most of it is fixed values const fixedStatus: Pick< IndexingStatus, | "type" @@ -384,6 +405,8 @@ export default class DocsService { url: siteIndexingConfig.startUrl, }; + // Clear current indexes if reIndexing + // if (indexExists) { if (reIndex) { await this.deleteIndexes(startUrl); @@ -399,6 +422,12 @@ export default class DocsService { } } + // If not preindexed + const isPreIndexedDoc = !!preIndexedDocs[siteIndexingConfig.startUrl]; + if (!isPreIndexedDoc) { + this.addToConfig(siteIndexingConfig); + } + try { this.handleStatusUpdate({ ...fixedStatus, @@ -436,7 +465,7 @@ export default class DocsService { articles.push(article); processedPages++; - await new Promise((resolve) => setTimeout(resolve, 50)); // Locks down GUI if no sleeping + await new Promise((resolve) => setTimeout(resolve, 100)); // Locks down GUI if no sleeping } void Telemetry.capture("docs_pages_crawled", { @@ -789,19 +818,13 @@ export default class DocsService { } } - // Sends "Pending" or current status for each - await this.initStatuses(); - - for (const doc of changedDocs) { - console.log(`Updating indexed doc: ${doc.startUrl}`); - await this.indexAndAdd(doc, true); - } + await Promise.allSettled([ + ...changedDocs.map((doc) => this.indexAndAdd(doc, true)), + ...newDocs.map((doc) => this.indexAndAdd(doc)), + ]); for (const doc of newDocs) { - console.log(`Indexing new doc: ${doc.startUrl}`); void Telemetry.capture("add_docs_config", { url: doc.startUrl }); - - await this.indexAndAdd(doc); } for (const doc of deletedDocs) { @@ -942,7 +965,7 @@ export default class DocsService { ); } - private addToConfig({ siteIndexingConfig }: AddParams) { + private addToConfig(siteIndexingConfig: SiteIndexingConfig) { // Handles the case where a user has manually added the doc to config.json // so it already exists in the file const doesDocExist = this.config.docs?.some( @@ -960,13 +983,6 @@ export default class DocsService { private async add(params: AddParams) { await this.addToLance(params); await this.addMetadataToSqlite(params); - - const isPreIndexedDoc = - !!preIndexedDocs[params.siteIndexingConfig.startUrl]; - - if (!isPreIndexedDoc) { - this.addToConfig(params); - } } // Delete methods @@ -1031,10 +1047,7 @@ export default class DocsService { console.log( `Reindexing non-preindexed docs with new embeddings provider: ${embeddingsProvider.id}`, ); - await this.initStatuses(); - for (const doc of docs) { - await this.indexAndAdd(doc, true); - } + await Promise.allSettled(docs.map((doc) => this.indexAndAdd(doc))); // Important that this only is invoked after we have successfully // cleared and reindex the docs so that the table cannot end up in an diff --git a/core/indexing/docs/suggestions/index.ts b/core/indexing/docs/suggestions/index.ts new file mode 100644 index 0000000000..db192489ad --- /dev/null +++ b/core/indexing/docs/suggestions/index.ts @@ -0,0 +1,134 @@ +import { + PackageDocsResult, + FilePathAndName, + PackageFilePathAndName, + IDE, + PackageDetails, + ParsedPackageInfo, +} from "../../.."; +import { walkDir } from "../../walkDir"; + +import { PythonPackageCrawler } from "./packageCrawlers/Python"; +import { NodePackageCrawler } from "./packageCrawlers/TsJs"; + +const PACKAGE_CRAWLERS = [NodePackageCrawler, PythonPackageCrawler]; + +export interface PackageCrawler { + packageRegistry: string; + getPackageFiles(files: FilePathAndName[]): PackageFilePathAndName[]; + parsePackageFile( + file: PackageFilePathAndName, + contents: string, + ): ParsedPackageInfo[]; + getPackageDetails(packageInfo: ParsedPackageInfo): Promise; +} + +export async function getAllSuggestedDocs(ide: IDE) { + const workspaceDirs = await ide.getWorkspaceDirs(); + const results = await Promise.all( + workspaceDirs.map((dir) => { + return walkDir(dir, ide); + }), + ); + const allPaths = results.flat(); // TODO only get files, not dirs. Not critical for now + const allFiles = allPaths.map((path) => ({ + path, + name: path.split(/[\\/]/).pop()!, + })); + + // Build map of language -> package files + const packageFilesByRegistry: Record = {}; + for (const Crawler of PACKAGE_CRAWLERS) { + const crawler = new Crawler(); + const packageFilePaths = crawler.getPackageFiles(allFiles); + packageFilesByRegistry[crawler.packageRegistry] = packageFilePaths; + } + + // Get file contents for all unique package files + const uniqueFilePaths = Array.from( + new Set( + Object.values(packageFilesByRegistry).flatMap((files) => + files.map((file) => file.path), + ), + ), + ); + const fileContentsArray = await Promise.all( + uniqueFilePaths.map(async (path) => { + const contents = await ide.readFile(path); + return { path, contents }; + }), + ); + const fileContents = new Map( + fileContentsArray.map(({ path, contents }) => [path, contents]), + ); + + // Parse package files and build map of language -> packages + const packagesByCrawler: Record = {}; + PACKAGE_CRAWLERS.forEach((Crawler) => { + const crawler = new Crawler(); + packagesByCrawler[crawler.packageRegistry] = []; + const packageFiles = packageFilesByRegistry[crawler.packageRegistry]; + packageFiles.forEach((file) => { + const contents = fileContents.get(file.path); + if (!contents) { + return; + } + const packages = crawler.parsePackageFile(file, contents); + packagesByCrawler[crawler.packageRegistry].push(...packages); + }); + }); + + // Deduplicate packages per language + // TODO - this is where you would allow docs for different versions + // by e.g. using "name-version" as the map key instead of just name + // For now have not allowed + const registries = Object.keys(packagesByCrawler); + registries.forEach((registry) => { + const packages = packagesByCrawler[registry]; + const uniquePackages = Array.from( + new Map(packages.map((pkg) => [pkg.name, pkg])).values(), + ); + packagesByCrawler[registry] = uniquePackages; + }); + + // Get documentation links for all packages + const allDocsResults: PackageDocsResult[] = []; + await Promise.all( + PACKAGE_CRAWLERS.map(async (Crawler) => { + const crawler = new Crawler(); + const packages = packagesByCrawler[crawler.packageRegistry]; + const docsByRegistry = await Promise.all( + packages.map(async (packageInfo) => { + try { + const details = await crawler.getPackageDetails(packageInfo); + if (!details.docsLink) { + return { + packageInfo, + error: `No documentation link found for ${packageInfo.name}`, + }; + } + return { + packageInfo, + details: { + ...details, + docsLink: details.docsLink, + docsLinkWarning: details.docsLink.includes("github.com") + ? "Github docs not supported, find the docs site" + : details.docsLink.includes("docs") + ? undefined + : "May not be a docs site, check the URL", + }, + }; + } catch (error) { + return { + packageInfo, + error: `Error getting package details for ${packageInfo.name}`, + }; + } + }), + ); + allDocsResults.push(...docsByRegistry); + }), + ); + return allDocsResults; +} diff --git a/core/indexing/docs/suggestions/packageCrawlers/Python.ts b/core/indexing/docs/suggestions/packageCrawlers/Python.ts new file mode 100644 index 0000000000..3a617568f4 --- /dev/null +++ b/core/indexing/docs/suggestions/packageCrawlers/Python.ts @@ -0,0 +1,63 @@ +import { PackageCrawler } from ".."; +import { + FilePathAndName, + PackageDetails, + PackageFilePathAndName, + ParsedPackageInfo, +} from "../../../.."; + +export class PythonPackageCrawler implements PackageCrawler { + packageRegistry = "pypi"; + + getPackageFiles(files: FilePathAndName[]): PackageFilePathAndName[] { + // For Python, we typically look for files like requirements.txt or Pipfile + return files + .filter( + (file) => file.name === "requirements.txt" || file.name === "Pipfile", + ) + .map((file) => ({ + ...file, + packageRegistry: "pypi", + })); + } + + parsePackageFile( + file: PackageFilePathAndName, + contents: string, + ): ParsedPackageInfo[] { + // Assume the fileContent is a string from a requirements.txt formatted file + return contents + .split("\n") + .map((line) => { + const [name, version] = line.split("=="); + return { name, version, packageFile: file, language: "py" }; + }) + .filter((pkg) => pkg.name && pkg.version); + } + + async getPackageDetails( + packageInfo: ParsedPackageInfo, + ): Promise { + // Fetch metadata from PyPI to find the documentation link + const response = await fetch( + `https://pypi.org/pypi/${packageInfo.name}/json`, + ); + if (!response.ok) { + throw new Error(`Could not fetch data for package ${packageInfo.name}`); + } + const data = await response.json(); + const homePage = data?.info?.home_page as string | undefined; + + return { + docsLink: + (data?.info?.project_urls?.Documentation as string | undefined) ?? + homePage, + title: data?.info?.name as string | undefined, + description: data?.info?.summary as string | undefined, + repo: + (data?.info?.project_urls?.Repository as string | undefined) ?? + homePage, + license: data?.info?.license as string | undefined, + }; + } +} diff --git a/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts new file mode 100644 index 0000000000..3a9ef96c2b --- /dev/null +++ b/core/indexing/docs/suggestions/packageCrawlers/TsJs.ts @@ -0,0 +1,77 @@ +import { PackageCrawler } from ".."; +import { + FilePathAndName, + PackageDetails, + PackageFilePathAndName, + ParsedPackageInfo, +} from "../../../.."; + +export class NodePackageCrawler implements PackageCrawler { + packageRegistry = "npm"; + + getPackageFiles(files: FilePathAndName[]): PackageFilePathAndName[] { + // For Javascript/TypeScript, we look for package.json file + return files + .filter((file) => file.name === "package.json") + .map((file) => ({ + ...file, + packageRegistry: this.packageRegistry, + })); + } + + parsePackageFile( + file: PackageFilePathAndName, + contents: string, + ): ParsedPackageInfo[] { + // Parse the package.json content + const jsonData = JSON.parse(contents) as Record; + const dependencies = Object.entries(jsonData.dependencies || {}).concat( + Object.entries(jsonData.devDependencies || {}), + ); + + // Filter out types packages and check if typescript is present + let foundTypes = false; + const filtered = dependencies.filter(([name, _]) => { + if (name.startsWith("@types/")) { + foundTypes = true; + return false; + } + if (name.includes("typescript")) { + foundTypes = true; + } + return true; + }); + return filtered.map(([name, version]) => ({ + name, + version, + packageFile: file, + language: foundTypes ? "ts" : "js", + })); + } + + async getPackageDetails( + packageInfo: ParsedPackageInfo, + ): Promise { + const { name } = packageInfo; + // Fetch metadata from the NPM registry to find the documentation link + const response = await fetch(`https://registry.npmjs.org/${name}`); + if (!response.ok) { + throw new Error(`Could not fetch data for package ${name}`); + } + const data = await response.json(); + + // const dependencies = Object.keys(packageContentData.dependencies || {}) + // .concat(Object.keys(packageContentData.devDependencies || {})); + // const usesTypescript = dependencies.includes("typescript"); + + return { + docsLink: data.homepage as string | undefined, + title: name, // package.json doesn't have specific title field + description: data.description as string | undefined, + repo: Array.isArray(data.repository) + ? (data.respository[0]?.url as string | undefined) + : undefined, + license: data.license as string | undefined, + }; + } +} diff --git a/core/protocol/core.ts b/core/protocol/core.ts index c757e5d347..88cb5642c9 100644 --- a/core/protocol/core.ts +++ b/core/protocol/core.ts @@ -9,7 +9,7 @@ import type { DiffLine, FileSymbolMap, IdeSettings, - IndexingStatusMap, + IndexingStatus, LLMFullCompletionOptions, MessageContent, ModelDescription, @@ -166,7 +166,8 @@ export type ToCoreFromIdeOrWebviewProtocol = { "indexing/reindex": [{ type: string; id: string }, void]; "indexing/abort": [{ type: string; id: string }, void]; "indexing/setPaused": [{ type: string; id: string; paused: boolean }, void]; - "indexing/initStatuses": [undefined, void]; + "docs/getSuggestedDocs": [undefined, void]; + "docs/initStatuses": [undefined, void]; addAutocompleteModel: [{ model: ModelDescription }, void]; diff --git a/core/protocol/passThrough.ts b/core/protocol/passThrough.ts index 40d537d41b..720a6f66e8 100644 --- a/core/protocol/passThrough.ts +++ b/core/protocol/passThrough.ts @@ -48,10 +48,11 @@ export const WEBVIEW_TO_CORE_PASS_THROUGH: (keyof ToCoreFromWebviewProtocol)[] = "index/forceReIndexFiles", "index/indexingProgressBarInitialized", // Docs, etc. - "indexing/initStatuses", "indexing/reindex", "indexing/abort", "indexing/setPaused", + "docs/getSuggestedDocs", + "docs/initStatuses", // "completeOnboarding", "addAutocompleteModel", @@ -74,4 +75,5 @@ export const CORE_TO_WEBVIEW_PASS_THROUGH: (keyof ToWebviewFromCoreProtocol)[] = "getWebviewHistoryLength", "signInToControlPlane", "openDialogMessage", + "docs/suggestions", ]; diff --git a/core/protocol/webview.ts b/core/protocol/webview.ts index 0dcdb047f6..fb6a8837c5 100644 --- a/core/protocol/webview.ts +++ b/core/protocol/webview.ts @@ -4,6 +4,7 @@ import type { ContextItemWithId, IndexingProgressUpdate, IndexingStatus, + PackageDocsResult, } from "../index.js"; export type ToWebviewFromIdeOrCoreProtocol = { @@ -25,4 +26,5 @@ export type ToWebviewFromIdeOrCoreProtocol = { getWebviewHistoryLength: [undefined, number]; signInToControlPlane: [undefined, void]; openDialogMessage: ["account", void]; + "docs/suggestions": [PackageDocsResult[], void]; }; diff --git a/extensions/vscode/package-lock.json b/extensions/vscode/package-lock.json index a57404715a..742211fb08 100644 --- a/extensions/vscode/package-lock.json +++ b/extensions/vscode/package-lock.json @@ -109,7 +109,7 @@ "cheerio": "^1.0.0-rc.12", "commander": "^12.0.0", "comment-json": "^4.2.3", - "dbinfoz": "^0.11.0", + "dbinfoz": "^0.14.0", "diff": "^7.0.0", "dotenv": "^16.4.5", "fastest-levenshtein": "^1.0.16", diff --git a/gui/package-lock.json b/gui/package-lock.json index 7d5bf06c99..1a9e4fe3ff 100644 --- a/gui/package-lock.json +++ b/gui/package-lock.json @@ -111,7 +111,7 @@ "cheerio": "^1.0.0-rc.12", "commander": "^12.0.0", "comment-json": "^4.2.3", - "dbinfoz": "^0.11.0", + "dbinfoz": "^0.14.0", "diff": "^7.0.0", "dotenv": "^16.4.5", "fastest-levenshtein": "^1.0.16", diff --git a/gui/src/App.tsx b/gui/src/App.tsx index 1e34e6e156..ca807a8764 100644 --- a/gui/src/App.tsx +++ b/gui/src/App.tsx @@ -8,7 +8,7 @@ import useSubmenuContextProviders from "./hooks/useSubmenuContextProviders"; import { useVscTheme } from "./hooks/useVscTheme"; import { AddNewModel, ConfigureProvider } from "./pages/AddNewModel"; import ConfigErrorPage from "./pages/config-error"; -import Edit from "./pages/Edit"; +import Edit from "./pages/edit"; import ErrorPage from "./pages/error"; import Chat from "./pages/gui"; import History from "./pages/history"; diff --git a/gui/src/components/dialogs/AddDocsDialog.tsx b/gui/src/components/dialogs/AddDocsDialog.tsx index 0981948869..00134a6f73 100644 --- a/gui/src/components/dialogs/AddDocsDialog.tsx +++ b/gui/src/components/dialogs/AddDocsDialog.tsx @@ -1,48 +1,94 @@ import { - CheckCircleIcon, - ChevronDownIcon, - ChevronUpIcon, + CheckIcon, + InformationCircleIcon, + PencilIcon, + PlusIcon, } from "@heroicons/react/24/outline"; -import { SiteIndexingConfig } from "core"; +import { IndexingStatus, PackageDocsResult, SiteIndexingConfig } from "core"; import { usePostHog } from "posthog-js/react"; -import { useContext, useLayoutEffect, useRef, useState } from "react"; +import { useContext, useLayoutEffect, useMemo, useRef, useState } from "react"; import { useDispatch, useSelector } from "react-redux"; -import { Button, HelperText, Input, lightGray, SecondaryButton } from ".."; +import { Input, SecondaryButton } from ".."; import { IdeMessengerContext } from "../../context/IdeMessenger"; import { setDialogMessage, setShowDialog, } from "../../redux/slices/uiStateSlice"; import { RootState } from "../../redux/store"; -import IndexingStatusViewer from "../indexing/IndexingStatus"; +import { ToolTip } from "../gui/Tooltip"; +import FileIcon from "../FileIcon"; +import DocsIndexingPeeks from "../indexing/DocsIndexingPeeks"; +import { updateIndexingStatus } from "../../redux/slices/stateSlice"; +import preIndexedDocs from "core/indexing/docs/preIndexedDocs"; function AddDocsDialog() { const posthog = usePostHog(); const dispatch = useDispatch(); - const ref = useRef(null); + const titleRef = useRef(null); + const urlRef = useRef(null); const [title, setTitle] = useState(""); const [startUrl, setStartUrl] = useState(""); const [faviconUrl, setFaviconUrl] = useState(""); - const [isOpen, setIsOpen] = useState(false); - - const [submittedConfig, setSubmittedConfig] = useState(); const ideMessenger = useContext(IdeMessengerContext); const indexingStatuses = useSelector( (store: RootState) => store.state.indexing.statuses, ); + const docsIndexingStatuses: IndexingStatus[] = useMemo(() => { + return Object.values(indexingStatuses).filter( + (status) => status.type === "docs" && status.status === "indexing", + ); + }, [indexingStatuses]); + + const docsSuggestions = useSelector( + (store: RootState) => store.state.docsSuggestions, + ); + + const configDocs = useSelector((store: RootState) => store.state.config.docs); + + const sortedDocsSuggestions = useMemo(() => { + const docsFromConfig = configDocs ?? []; + const filtered = docsSuggestions.filter((sug) => { + const startUrl = sug.details?.docsLink; + return ( + !docsFromConfig.find((d) => d.startUrl === startUrl) && + !docsIndexingStatuses.find((s) => s.id === startUrl) && + (startUrl ? !preIndexedDocs[startUrl] : true) + ); + }); + + filtered.sort((a, b) => { + const rank = (result: PackageDocsResult) => { + if (result.error) { + return 2; + } else if (result.details?.docsLinkWarning) { + return 1; + } else { + return 0; + } + }; + return rank(a) - rank(b); + }); + return filtered; + }, [docsSuggestions, configDocs, docsIndexingStatuses]); + const isFormValid = startUrl && title; useLayoutEffect(() => { setTimeout(() => { - if (ref.current) { - ref.current.focus(); + if (titleRef.current) { + titleRef.current.focus(); } }, 100); - }, [ref]); + }, [titleRef]); + + const closeDialog = () => { + dispatch(setShowDialog(false)); + dispatch(setDialogMessage(undefined)); + }; function onSubmit(e: any) { e.preventDefault(); @@ -55,158 +101,228 @@ function AddDocsDialog() { ideMessenger.post("context/addDocs", siteIndexingConfig); - setSubmittedConfig(siteIndexingConfig); setTitle(""); setStartUrl(""); setFaviconUrl(""); posthog.capture("add_docs_gui", { url: startUrl }); - } - if (submittedConfig) { - const status = indexingStatuses[submittedConfig.startUrl]; - return ( -
-
- -

{`Docs added`}

-
-
-

Title: {submittedConfig.title}

-

Start URL: {submittedConfig.startUrl}

- {submittedConfig.rootUrl && ( -

Root URL: {submittedConfig.rootUrl}

- )} - {submittedConfig.maxDepth && ( -

Max depth: {submittedConfig.maxDepth}

- )} - {submittedConfig.faviconUrl && ( -

Favicon URL: {submittedConfig.faviconUrl}

- )} -
- {!!status && ( -
-

{`Type "@docs" and select ${submittedConfig.title} to reference these docs once indexing is complete. Check indexing status from the "More" page.`}

-
- -
-
- )} -
- { - setSubmittedConfig(undefined); - }} - > - Add another - - -
-
+ // Optimistic status update + dispatch( + updateIndexingStatus({ + type: "docs", + description: "Initializing", + id: startUrl, + embeddingsProviderId: "mock-embeddings-provider-id", + progress: 0, + status: "indexing", + title, + url: startUrl, + }), ); } - return ( -
-
-

Add a documentation site

+ const handleSelectSuggestion = (docsResult: PackageDocsResult) => { + if (docsResult.error) { + setTitle(docsResult.packageInfo.name); + setStartUrl(""); + urlRef.current?.focus(); + return; + } + if (docsResult.details?.docsLinkWarning) { + setTitle(docsResult.packageInfo.name); + setStartUrl(docsResult.details.docsLink); + urlRef.current?.focus(); + return; + } + const siteIndexingConfig: SiteIndexingConfig = { + startUrl: docsResult.details.docsLink, + title: docsResult.details.title, + faviconUrl: undefined, + }; -

- Continue pre-indexes many common documentation sites, but if there's - one you don't see in the dropdown, enter the URL here. -

+ ideMessenger.post("context/addDocs", siteIndexingConfig); -

- Continue's indexing engine will crawl the site and generate embeddings - so that you can ask questions. -

-
+ posthog.capture("add_docs_gui", { url: startUrl }); -
- - - - -
setIsOpen((prev) => !prev)} - > - {isOpen ? ( - - ) : ( - - )} - Advanced -
+ // Optimistic status update + dispatch( + updateIndexingStatus({ + type: "docs", + description: "Initializing", + id: docsResult.details.docsLink, + embeddingsProviderId: "mock-embeddings-provider-id", + progress: 0, + status: "indexing", + title: docsResult.details.title ?? docsResult.packageInfo.name, + url: docsResult.details.docsLink, + }), + ); + }; - {isOpen && ( -
- -
+ return ( +
+
+

Add documentation

+

Add docs

+

+ For the @docs context provider +

+ {!!sortedDocsSuggestions.length && ( +

Suggestions

)} +
+ {sortedDocsSuggestions.map((docsResult) => { + const { error, details } = docsResult; + const { language, name, version } = docsResult.packageInfo; + const id = `${language}-${name}-${version}`; + return ( +
{ + handleSelectSuggestion(docsResult); + }} + > +
+ {error || details.docsLinkWarning ? ( +
+ + + This may not be a docs page + +
+ ) : ( + + )} +
+
+
+ +
+ {name} +
+
+ {error ? ( + + No docs link found + + ) : ( +
+ {/*
+ +
*/} +

+ {details.docsLink} +

+
+ )} +
+
+ + +

{`Version: ${version}`}

+

{`Found in ${docsResult.packageInfo.packageFile.path}`}

+ +
+
+ ); + })} +
+
+ +
+ -
- + + {/*
+ +
*/} +
+
+ + Add + +
+
- +
+ + +
+
+ {docsIndexingStatuses.length ? ( +

+ + It is safe to close this form while indexing +

+ ) : null} +
+ {/* */} +
); } diff --git a/gui/src/components/dialogs/index.tsx b/gui/src/components/dialogs/index.tsx index 7685109baf..8a3a67fe84 100644 --- a/gui/src/components/dialogs/index.tsx +++ b/gui/src/components/dialogs/index.tsx @@ -71,7 +71,7 @@ const TextDialog = (props: TextDialogProps) => { return (