Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Autodetect Docs V1 #3038

Merged
merged 21 commits into from
Nov 26, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions core/config/ProfileLifecycleManager.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
import { config } from "dotenv";

import {
BrowserSerializedContinueConfig,
ContinueConfig,
1 change: 1 addition & 0 deletions core/config/load.ts
Original file line number Diff line number Diff line change
@@ -527,6 +527,7 @@ function finalToBrowserConfig(
embeddingsProvider: final.embeddingsProvider?.id,
ui: final.ui,
experimental: final.experimental,
docs: final.docs,
};
}

6 changes: 6 additions & 0 deletions core/core.ts
Original file line number Diff line number Diff line change
@@ -735,6 +735,10 @@ export class Core {
return this.docsService.initStatuses();
});
on("docs/getSuggestedDocs", async (msg) => {
if (hasRequestedDocs) {
return;
} // TODO, remove, hack because of rerendering
hasRequestedDocs = true;
const suggestedDocs = await getAllSuggestedDocs(this.ide);
this.messenger.send("docs/suggestions", suggestedDocs);
});
@@ -842,3 +846,5 @@ export class Core {

// private
}

let hasRequestedDocs = false;
2 changes: 2 additions & 0 deletions core/index.d.ts
Original file line number Diff line number Diff line change
@@ -1189,6 +1189,7 @@ export interface BrowserSerializedContinueConfig {
reranker?: RerankerDescription;
experimental?: ExperimentalConfig;
analytics?: AnalyticsConfig;
docs?: SiteIndexingConfig[];
}

// DOCS SUGGESTIONS AND PACKAGE INFO
@@ -1210,6 +1211,7 @@ export type ParsedPackageInfo = {

export type PackageDetails = {
docsLink?: string;
docsLinkWarning?: string;
title?: string;
description?: string;
repo?: string;
39 changes: 13 additions & 26 deletions core/indexing/docs/suggestions/index.ts
Original file line number Diff line number Diff line change
@@ -63,42 +63,40 @@ export async function getAllSuggestedDocs(ide: IDE) {
);

// Parse package files and build map of language -> packages
const packagesByRegistry: Record<string, ParsedPackageInfo[]> = {};
const packagesByCrawler: Record<string, ParsedPackageInfo[]> = {};
PACKAGE_CRAWLERS.forEach((Crawler) => {
const crawler = new Crawler();
packagesByCrawler[crawler.packageRegistry] = [];
const packageFiles = packageFilesByRegistry[crawler.packageRegistry];
packageFiles.forEach((file) => {
const contents = fileContents.get(file.path);
if (!contents) {
return;
}
const packages = crawler.parsePackageFile(file, contents);
if (!packagesByRegistry[crawler.packageRegistry]) {
packagesByRegistry[crawler.packageRegistry] = [];
}
packagesByRegistry[crawler.packageRegistry].push(...packages);
packagesByCrawler[crawler.packageRegistry].push(...packages);
});
});

// Deduplicate packages per language
// TODO - this is where you would allow docs for different versions
// by e.g. using "name-version" as the map key instead of just name
// For now have not allowed
const languages = Object.keys(packagesByRegistry);
languages.forEach((language) => {
const packages = packagesByRegistry[language];
const registries = Object.keys(packagesByCrawler);
registries.forEach((registry) => {
const packages = packagesByCrawler[registry];
const uniquePackages = Array.from(
new Map(packages.map((pkg) => [pkg.name, pkg])).values(),
);
packagesByRegistry[language] = uniquePackages;
packagesByCrawler[registry] = uniquePackages;
});

// Get documentation links for all packages
const allDocsResults: PackageDocsResult[] = [];
await Promise.all(
PACKAGE_CRAWLERS.map(async (Crawler) => {
const crawler = new Crawler();
const packages = packagesByRegistry[crawler.packageRegistry];
const packages = packagesByCrawler[crawler.packageRegistry];
const docsByRegistry = await Promise.all(
packages.map(async (packageInfo) => {
try {
@@ -114,6 +112,11 @@ export async function getAllSuggestedDocs(ide: IDE) {
details: {
...details,
docsLink: details.docsLink,
docsLinkWarning: details.docsLink.includes("github.com")
? "Github docs not supported, find the docs site"
: details.docsLink.includes("docs")
? undefined
: "May not be a docs site, check the URL",
},
};
} catch (error) {
@@ -129,19 +132,3 @@ export async function getAllSuggestedDocs(ide: IDE) {
);
return allDocsResults;
}

// write me an interface PackageCrawler that contains:
// 1. property `language` to store a given language like "python" or "typescript"
// 2. has a method `getPackageFiles` which takes a list of file names and decides which ones match package/dependency files (e.g. package.json for typescript, requirements.txt for python, etc)
// 3. has a method `parsePackageFile` which returns a list of package name and version from a relevant package file, in a standardized format like semver
// 4. has a method `getDocumentationLink` to check for documentation link for a given package (e.g. GET `https://registry.npmjs.org/<package>` and find docs field for typescript, documentation link in the package metadata for PyPi, etc.)
// Then, write typescript classes to implement this typescript interface for the languages "python" and "typescript"

// I want to present the user with a list of dependencies and allow them to select which ones to index (embed) documentation for.
// In order to prevent duplicate file reads, the process will be like this:
// 1. take in a list of filepaths called `filepaths`
// 2. loop an array of PackageCrawler classes to build a map of `language` (string) to `packageFilePaths` (string[])
// 3. Get unique filepaths from `packageFilePaths` and build a map ` of filepath to file contents using an existing `readFile` function, and skipping file reads of already in the map
// Finally,
// Add a `` method to the interface and classes that returns
// Then, assemble the classes in an array, and write a function getAllSuggestedDocs that returns a map of `language` to an ar
2 changes: 1 addition & 1 deletion core/indexing/docs/suggestions/packageCrawlers/Python.ts
Original file line number Diff line number Diff line change
@@ -30,7 +30,7 @@ export class PythonPackageCrawler implements PackageCrawler {
.split("\n")
.map((line) => {
const [name, version] = line.split("==");
return { name, version, packageFile: file, language: "python" };
return { name, version, packageFile: file, language: "py" };
})
.filter((pkg) => pkg.name && pkg.version);
}
1 change: 0 additions & 1 deletion core/protocol/core.ts
Original file line number Diff line number Diff line change
@@ -9,7 +9,6 @@ import type {
DiffLine,
FileSymbolMap,
IdeSettings,
IndexingStatusMap,
LLMFullCompletionOptions,
MessageContent,
ModelDescription,
4 changes: 2 additions & 2 deletions core/protocol/webview.ts
Original file line number Diff line number Diff line change
@@ -2,9 +2,9 @@ import { ConfigValidationError } from "../config/validation.js";

import type {
ContextItemWithId,
DocsSuggestions,
IndexingProgressUpdate,
IndexingStatus,
PackageDocsResult,
} from "../index.js";

export type ToWebviewFromIdeOrCoreProtocol = {
@@ -26,5 +26,5 @@ export type ToWebviewFromIdeOrCoreProtocol = {
getWebviewHistoryLength: [undefined, number];
signInToControlPlane: [undefined, void];
openDialogMessage: ["account", void];
"docs/suggestions": [DocsSuggestions, void];
"docs/suggestions": [PackageDocsResult[], void];
};
469 changes: 264 additions & 205 deletions gui/src/components/dialogs/AddDocsDialog.tsx

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion gui/src/components/dialogs/index.tsx
Original file line number Diff line number Diff line change
@@ -71,7 +71,7 @@ const TextDialog = (props: TextDialogProps) => {
return (
<ScreenCover onClick={props.onClose} hidden={!props.showDialog}>
<DialogContainer
className="xs:w-[85%] w-[92%] sm:w-[75%]"
className="xs:w-[90%] no-scrollbar max-h-full w-[92%] max-w-[600px] overflow-auto sm:w-[88%] md:w-[80%]"
onClick={(e) => {
e.stopPropagation();
}}
2 changes: 1 addition & 1 deletion gui/src/components/indexing/ChatIndexingPeeks.tsx
Original file line number Diff line number Diff line change
@@ -104,7 +104,7 @@ function ChatIndexingPeeks() {
return (
<div className="flex flex-col gap-1">
{mergedIndexingStates.map((state) => {
return <ChatIndexingPeek state={state} />;
return <ChatIndexingPeek key={state.type} state={state} />;
})}
</div>
);
76 changes: 76 additions & 0 deletions gui/src/components/indexing/DocsIndexingPeeks.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
import { useDispatch, useSelector } from "react-redux";
import { RootState } from "../../redux/store";

import { IndexingStatus } from "core";
import { useMemo } from "react";
import { useNavigate } from "react-router-dom";
import { ArrowPathIcon, EyeSlashIcon } from "@heroicons/react/24/outline";
import {
setDialogMessage,
setShowDialog,
} from "../../redux/slices/uiStateSlice";

export interface DocsIndexingPeekProps {
status: IndexingStatus;
}

function DocsIndexingPeek({ status }: DocsIndexingPeekProps) {
const navigate = useNavigate();
const dispatch = useDispatch();

const progressPercentage = useMemo(() => {
return Math.min(100, Math.max(0, status.progress * 100));
}, [status.progress]);

return (
<div
className="flex cursor-pointer flex-row items-center gap-2 rounded-md px-1 text-stone-500 hover:bg-gray-700/10"
onClick={() => {
navigate("/more");
dispatch(setShowDialog(false));
dispatch(setDialogMessage(undefined));
}}
>
<p className="m-0 p-0 text-stone-500 group-hover:underline">
{status.title}
</p>
<div className="my-2 h-1.5 flex-1 rounded-md border border-solid border-gray-400">
<div
className={`h-full rounded-lg bg-stone-500 transition-all duration-200 ease-in-out`}
style={{
width: `${progressPercentage}%`,
}}
/>
</div>
<div className="xs:flex hidden flex-row items-center gap-1 text-stone-500">
<span className="text-xs no-underline">
{progressPercentage.toFixed(0)}%
</span>
<ArrowPathIcon
className={`animate-spin-slow inline-block h-4 w-4 text-stone-500`}
></ArrowPathIcon>
</div>
</div>
);
}

interface DocsIndexingPeeksProps {
statuses: IndexingStatus[];
}

function DocsIndexingPeekList({ statuses }: DocsIndexingPeeksProps) {
if (!statuses.length) return null;

return (
<div className="flex flex-col">
<p className="mx-0 my-1.5 p-0 text-stone-500">Currently Indexing</p>
<div className="max-h-[100px] overflow-y-auto pr-2">
{statuses.map((status) => {
return <DocsIndexingPeek key={status.id} status={status} />;
})}
</div>
</div>
);
}

export default DocsIndexingPeekList;
41 changes: 24 additions & 17 deletions gui/src/components/indexing/IndexingStatuses.tsx
Original file line number Diff line number Diff line change
@@ -8,6 +8,8 @@ import {
setShowDialog,
} from "../../redux/slices/uiStateSlice";
import AddDocsDialog from "../dialogs/AddDocsDialog";
import { PlusCircleIcon } from "@heroicons/react/24/outline";
import { ToolTip } from "../gui/Tooltip";

function IndexingStatuses() {
const indexingStatuses = useSelector(
@@ -25,30 +27,35 @@ function IndexingStatuses() {
<div className="flex flex-col gap-1">
<div className="flex flex-row items-center justify-between">
<h3 className="mb-1 mt-0 text-xl">@docs indexes</h3>
{/* <div className="border-1 rounded-full border">
<ChevronUpIcon className="h-8 w-8" />
</div> */}
{/* TODO add some way to hide, scroll, etc. */}
{docsStatuses.length ? (
<>
<PlusCircleIcon
data-tooltip-id={"more-add-docs-button"}
className="text-vsc-foreground-muted h-5 w-5 cursor-pointer focus:ring-0"
onClick={() => {
dispatch(setShowDialog(true));
dispatch(setDialogMessage(<AddDocsDialog />));
}}
/>
<ToolTip id={"more-add-docs-button"} place="top">
Add Docs
</ToolTip>
</>
) : null}
</div>
<span className="text-xs text-stone-500">
Manage your documentation sources
</span>
{/* <div className="flex max-h-[170px] flex-col gap-1 overflow-x-hidden overflow-y-scroll pr-2"> */}
{docsStatuses.length
? docsStatuses.map((status) => {
<div className="flex max-h-[170px] flex-col gap-1 overflow-x-hidden overflow-y-scroll pr-2">
{docsStatuses.length ? (
docsStatuses.map((status) => {
return <IndexingStatusViewer key={status.id} status={status} />;
})
: null}
<SecondaryButton
onClick={() => {
dispatch(setShowDialog(true));
dispatch(setDialogMessage(<AddDocsDialog />));
}}
>
Add Docs
</SecondaryButton>
) : (
<SecondaryButton>Add Docs</SecondaryButton>
)}
</div>
</div>
// </div>
);
}