Merge branch 'dev' of https://github.com/continuedev/continue into dev

continuedev · Sep 25, 2024 · 5b04828 · 5b04828
2 parents 14b6d68 + aa46bd7
commit 5b04828
Show file tree

Hide file tree

Showing 32 changed files with 471 additions and 658 deletions.
diff --git a/core/commands/util.ts b/core/commands/util.ts
@@ -1,4 +1,4 @@
-import { ContextItemWithId } from "../index.js";
+import { ContextItemWithId } from "../";
 
 export interface RangeInFileWithContents {
   filepath: string;
@@ -15,7 +15,7 @@ export function contextItemToRangeInFileWithContents(
   const lines = item.name.split("(")[1].split(")")[0].split("-");
 
   const rif: RangeInFileWithContents = {
-    filepath: item.id.itemId,
+    filepath: item.uri?.value || "",
     range: {
       start: {
         line: Number.parseInt(lines[0]),

diff --git a/core/config/default.ts b/core/config/default.ts
@@ -23,7 +23,9 @@ export const FREE_TRIAL_MODELS: ModelDescription[] = [
   {
     title: "Claude 3 Sonnet (Free Trial)",
     provider: "free-trial",
-    model: "claude-3-sonnet-20240229",
+    model: "claude-3-5-sonnet-20240620",
+    systemMessage:
+      "You are an expert software developer. You give helpful and concise responses.",
   },
   {
     title: "GPT-4o (Free Trial)",
@@ -35,14 +37,16 @@ export const FREE_TRIAL_MODELS: ModelDescription[] = [
   {
     title: "Llama3 70b (Free Trial)",
     provider: "free-trial",
-    model: "llama3-70b",
+    model: "llama3.1-70b",
     systemMessage:
-      "You are an expert software developer. You give helpful and concise responses. Whenever you write a code block you include the language after the opening ticks.",
+      "You are an expert software developer. You give helpful and concise responses.",
   },
   {
     title: "Codestral (Free Trial)",
     provider: "free-trial",
-    model: "codestral",
+    model: "codestral-latest",
+    systemMessage:
+      "You are an expert software developer. You give helpful and concise responses.",
   },
 ];
 

diff --git a/core/context/providers/CurrentFileContextProvider.ts b/core/context/providers/CurrentFileContextProvider.ts
@@ -2,9 +2,9 @@ import {
   ContextItem,
   ContextProviderDescription,
   ContextProviderExtras,
-} from "../../index.js";
-import { getBasename } from "../../util/index.js";
-import { BaseContextProvider } from "../index.js";
+} from "../../";
+import { getBasename } from "../../util/";
+import { BaseContextProvider } from "../";
 
 class CurrentFileContextProvider extends BaseContextProvider {
   static description: ContextProviderDescription = {
@@ -32,6 +32,10 @@ class CurrentFileContextProvider extends BaseContextProvider {
           currentFile,
         )}\n${contents}\n\`\`\``,
         name: getBasename(currentFile),
+        uri: {
+          type: "file",
+          value: currentFile,
+        },
       },
     ];
   }

diff --git a/core/context/providers/DocsContextProvider.ts b/core/context/providers/DocsContextProvider.ts
@@ -91,7 +91,7 @@ class DocsContextProvider extends BaseContextProvider {
       await docsService.isJetBrainsAndPreIndexedDocsProvider();
 
     if (isJetBrainsAndPreIndexedDocsProvider) {
-      extras.ide.showToast(
+      await extras.ide.showToast(
         "error",
         `${DocsService.preIndexedDocsEmbeddingsProvider.id} is configured as ` +
           "the embeddings provider, but it cannot be used with JetBrains. " +
@@ -105,7 +105,7 @@ class DocsContextProvider extends BaseContextProvider {
     const preIndexedDoc = preIndexedDocs[query];
 
     if (!!preIndexedDoc) {
-      Telemetry.capture("docs_pre_indexed_doc_used", {
+      void Telemetry.capture("docs_pre_indexed_doc_used", {
         doc: preIndexedDoc["title"],
       });
     }
@@ -147,6 +147,10 @@ class DocsContextProvider extends BaseContextProvider {
             : chunk.otherMetadata?.title || chunk.filepath,
           description: chunk.filepath,
           content: chunk.content,
+          uri: {
+            type: "url" as const,
+            value: chunk.filepath,
+          },
         }))
         .reverse(),
       {

diff --git a/core/context/providers/FileContextProvider.ts b/core/context/providers/FileContextProvider.ts
@@ -36,6 +36,10 @@ class FileContextProvider extends BaseContextProvider {
         name: query.split(/[\\/]/).pop() ?? query,
         description: query,
         content: `\`\`\`${query}\n${content}\n\`\`\``,
+        uri: {
+          type: "file",
+          value: query,
+        },
       },
     ];
   }

diff --git a/core/context/providers/OpenFilesContextProvider.ts b/core/context/providers/OpenFilesContextProvider.ts
@@ -28,10 +28,15 @@ class OpenFilesContextProvider extends BaseContextProvider {
       openFiles.map(async (filepath: string) => {
         return {
           description: filepath,
-          content: `\`\`\`${await getRelativePath(filepath, workspaceDirs)}\n${await ide.readFile(
+          content: `\`\`\`${await getRelativePath(
             filepath,
-          )}\n\`\`\``,
+            workspaceDirs,
+          )}\n${await ide.readFile(filepath)}\n\`\`\``,
           name: (filepath.split("/").pop() ?? "").split("\\").pop() ?? "",
+          uri: {
+            type: "file",
+            value: filepath,
+          },
         };
       }),
     );

diff --git a/core/context/providers/URLContextProvider.ts b/core/context/providers/URLContextProvider.ts
@@ -1,12 +1,13 @@
 import { Readability } from "@mozilla/readability";
 import { JSDOM } from "jsdom";
 import { NodeHtmlMarkdown } from "node-html-markdown";
-import { BaseContextProvider } from "../index.js";
+import { BaseContextProvider } from "../";
 import {
   ContextItem,
   ContextProviderDescription,
   ContextProviderExtras,
 } from "../../index.js";
+import { fetchFavicon } from "../../util/fetchFavicon";
 
 class URLContextProvider extends BaseContextProvider {
   static description: ContextProviderDescription = {
@@ -22,6 +23,7 @@ class URLContextProvider extends BaseContextProvider {
   ): Promise<ContextItem[]> {
     try {
       const url = new URL(query);
+      const icon = await fetchFavicon(url);
       const resp = await extras.fetch(url);
       const html = await resp.text();
 
@@ -37,11 +39,17 @@ class URLContextProvider extends BaseContextProvider {
       );
 
       const title = article?.title || url.pathname;
+
       return [
         {
-          description: title,
+          icon,
+          description: url.toString(),
           content: markdown,
           name: title,
+          uri: {
+            type: "url",
+            value: url.toString(),
+          },
         },
       ];
     } catch (e) {

diff --git a/core/context/retrieval/retrieval.ts b/core/context/retrieval/retrieval.ts
@@ -6,6 +6,7 @@ import { INSTRUCTIONS_BASE_ITEM } from "../providers/utils";
 import { RetrievalPipelineOptions } from "./pipelines/BaseRetrievalPipeline";
 import NoRerankerRetrievalPipeline from "./pipelines/NoRerankerRetrievalPipeline";
 import RerankerRetrievalPipeline from "./pipelines/RerankerRetrievalPipeline";
+import path from "path";
 
 export async function retrieveContextItemsFromEmbeddings(
   extras: ContextProviderExtras,
@@ -94,21 +95,32 @@ export async function retrieveContextItemsFromEmbeddings(
   }
 
   return [
-    ...results.map((r) => {
-      const name = `${getRelativePath(r.filepath, workspaceDirs)} (${
-        r.startLine
-      }-${r.endLine})`;
-      const description = `${r.filepath} (${r.startLine}-${r.endLine})`;
-      return {
-        name,
-        description,
-        content: `\`\`\`${name}\n${r.content}\n\`\`\``,
-      };
-    }),
     {
       ...INSTRUCTIONS_BASE_ITEM,
       content:
         "Use the above code to answer the following question. You should not reference any files outside of what is shown, unless they are commonly known files, like a .gitignore or package.json. Reference the filenames whenever possible. If there isn't enough information to answer the question, suggest where the user might look to learn more.",
     },
+    ...results
+      .sort((a, b) => a.filepath.localeCompare(b.filepath))
+      .map((r) => {
+        const name = `${path.basename(r.filepath)} (${r.startLine}-${
+          r.endLine
+        })`;
+        const description = `${r.filepath}`;
+
+        if (r.filepath.includes("package.json")) {
+          console.log();
+        }
+
+        return {
+          name,
+          description,
+          content: `\`\`\`${name}\n${r.content}\n\`\`\``,
+          uri: {
+            type: "file" as const,
+            value: r.filepath,
+          },
+        };
+      }),
   ];
 }
diff --git a/core/index.d.ts b/core/index.d.ts
@@ -271,23 +271,25 @@ export interface ContextItemId {
   itemId: string;
 }
 
+export type ContextItemUriTypes = "file" | "url";
+
+export interface ContextItemUri {
+  type: ContextItemUriTypes;
+  value: string;
+}
+
 export interface ContextItem {
   content: string;
   name: string;
   description: string;
   editing?: boolean;
   editable?: boolean;
   icon?: string;
+  uri?: ContextItemUri;
 }
 
-export interface ContextItemWithId {
-  content: string;
-  name: string;
-  description: string;
+export interface ContextItemWithId extends ContextItem {
   id: ContextItemId;
-  editing?: boolean;
-  editable?: boolean;
-  icon?: string;
 }
 
 export interface InputModifiers {

diff --git a/core/indexing/CodeSnippetsIndex.ts b/core/indexing/CodeSnippetsIndex.ts
@@ -341,6 +341,10 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex {
       name: row.title,
       description: getLastNPathParts(row.path, 2),
       content: `\`\`\`${getBasename(row.path)}\n${row.content}\n\`\`\``,
+      uri: {
+        type: "file",
+        value: row.path,
+      },
     };
   }
 

diff --git a/core/indexing/docs/DocsService.ts b/core/indexing/docs/DocsService.ts
@@ -32,6 +32,7 @@ import {
   SiteIndexingResults,
 } from "./preIndexed";
 import preIndexedDocs from "./preIndexedDocs";
+import { fetchFavicon, getFaviconBase64 } from "../../util/fetchFavicon";
 
 // Purposefully lowercase because lancedb converts
 export interface LanceDbDocsRow {
@@ -154,7 +155,10 @@ export default class DocsService {
         params: {},
       });
 
-      this.ide.showToast("info", "Successfuly added docs context provider");
+      await this.ide.showToast(
+        "info",
+        "Successfuly added docs context provider",
+      );
     }
 
     return res === actionMsg;
@@ -177,7 +181,7 @@ export default class DocsService {
       while (!(await generator.next()).done) {}
     }
 
-    this.ide.showToast("info", "Docs indexing completed");
+    await this.ide.showToast("info", "Docs indexing completed");
   }
 
   async list() {
@@ -250,7 +254,7 @@ export default class DocsService {
       }
     }
 
-    Telemetry.capture("docs_pages_crawled", {
+    void Telemetry.capture("docs_pages_crawled", {
       count: processedPages,
     });
 
@@ -321,7 +325,7 @@ export default class DocsService {
       await this.delete(startUrl);
     }
 
-    const favicon = await this.fetchFavicon(siteIndexingConfig);
+    const favicon = await fetchFavicon(new URL(siteIndexingConfig.startUrl));
 
     await this.add({
       siteIndexingConfig,
@@ -460,7 +464,7 @@ export default class DocsService {
 
     for (const doc of newDocs) {
       console.log(`Indexing new doc: ${doc.startUrl}`);
-      Telemetry.capture("add_docs_config", { url: doc.startUrl });
+      void Telemetry.capture("add_docs_config", { url: doc.startUrl });
 
       const generator = this.indexAndAdd(doc);
       while (!(await generator.next()).done) {}
@@ -537,8 +541,9 @@ export default class DocsService {
   private async getLanceTableNameFromEmbeddingsProvider(
     isPreIndexedDoc: boolean,
   ) {
-    const embeddingsProvider =
-      await this.getEmbeddingsProvider(isPreIndexedDoc);
+    const embeddingsProvider = await this.getEmbeddingsProvider(
+      isPreIndexedDoc,
+    );
     const embeddingsProviderId = this.removeInvalidLanceTableNameChars(
       embeddingsProvider.id,
     );
@@ -694,7 +699,12 @@ export default class DocsService {
 
     const siteEmbeddings = JSON.parse(data) as SiteIndexingResults;
     const startUrl = new URL(siteEmbeddings.url).toString();
-    const favicon = await this.fetchFavicon(preIndexedDocs[startUrl]);
+
+    const faviconUrl = preIndexedDocs[startUrl].faviconUrl;
+    const favicon =
+      typeof faviconUrl === "string"
+        ? await getFaviconBase64(faviconUrl)
+        : undefined;
 
     await this.add({
       favicon,
@@ -707,30 +717,6 @@ export default class DocsService {
     });
   }
 
-  private async fetchFavicon(siteIndexingConfig: SiteIndexingConfig) {
-    const faviconUrl =
-      siteIndexingConfig.faviconUrl ??
-      new URL("/favicon.ico", siteIndexingConfig.startUrl);
-
-    try {
-      const response = await fetch(faviconUrl);
-      const arrayBuffer = await response.arrayBuffer();
-      const base64 = btoa(
-        new Uint8Array(arrayBuffer).reduce(
-          (data, byte) => data + String.fromCharCode(byte),
-          "",
-        ),
-      );
-      const mimeType = response.headers.get("content-type") || "image/x-icon";
-
-      return `data:${mimeType};base64,${base64}`;
-    } catch {
-      console.error(`Failed to fetch favicon: ${faviconUrl}`);
-    }
-
-    return undefined;
-  }
-
   private async shouldReindexDocsOnNewEmbeddingsProvider(
     curEmbeddingsProviderId: EmbeddingsProvider["id"],
   ): Promise<boolean> {