Merge branch 'preview'

.changes/unreleased/Added-20240124-222904.yaml

@@ -0,0 +1,4 @@
+project: extensions/vscode
+kind: Added
+body: Image support
+time: 2024-01-24T22:29:04.193864-08:00

.gitignore

@@ -149,4 +149,6 @@ codeql
 **/**/build
 
 continue_server.build
-continue_server.dist
+continue_server.dist
+
+Icon

.vscode/tasks.json

@@ -102,8 +102,9 @@
     {
       "label": "install-all-dependencies",
       "type": "shell",
+      "windows": { "command": "./install-dependencies.ps1" },
       "command": "./install-dependencies.sh",
-      "problemMatcher": [] // Empty so users are not promted to select progress reporting
+      "problemMatcher": [] // Empty so users are not prompted to select progress reporting
     },
     //
     // Start the React App for debugging with Vite

CONTRIBUTING.md

@@ -57,7 +57,7 @@ Continue is continuously improving, but a feature isn't complete until it is ref
 
 VS Code is assumed for development as Continue is primarily a VS Code tool at the moment. Most of the setup and running is automated through VS Code tasks and launch configurations.
 
-Pre-requisite: you will need `cargo` the rust package manager installed ([get it on rust-lang.org](https://www.rust-lang.org/tools/install)).
+<!-- Pre-requisite: you will need `cargo` the rust package manager installed ([get it on rust-lang.org](https://www.rust-lang.org/tools/install)). -->
 
 1. Clone and open in VS Code the Continue repo `https://github.com/continuedev/continue`
 
@@ -83,11 +83,11 @@ Continue uses [Prettier](https://marketplace.visualstudio.com/items?itemName=esb
 
 ### Writing Slash Commands
 
-A Step can be used as a custom slash command, or called otherwise in a `Policy`. See the [steps README](./server/continuedev/plugins/steps/README.md) to learn how to write a Step.
+See the [slash command documentation](./docs/docs/customization/slash-commands.md) to learn how to write a custom slash command.
 
 ### Writing Context Providers
 
-A `ContextProvider` is a Continue plugin that lets type '@' to quickly select documents as context for the language model. The simplest way to create a `ContextProvider` is to implement the `provide_context_items` method. You can find a great example of this in [GitHubIssuesContextProvider](./server/continuedev/plugins/context_providers/github.py), which allows you to search GitHub Issues in a repo.
+A `ContextProvider` is a Continue plugin that lets type '@' to quickly select documents as context for the language model. The simplest way to create a `ContextProvider` is to implement the `provide_context_items` method. You can find a great example of this in [GitHubIssuesContextProvider](./core/context/providers/GitHubIssuesContextProvider.ts), which allows you to search GitHub Issues in a repo.
 
 ### Adding an LLM Provider
 

core/commands/index.ts

@@ -1,4 +1,5 @@
 import { CustomCommand, SlashCommand, SlashCommandDescription } from "..";
+import { stripImages } from "../llm/countTokens";
 import SlashCommands from "./slash";
 
 export function slashFromCustomCommand(
@@ -14,15 +15,15 @@ export function slashFromCustomCommand(
       for (let i = messages.length - 1; i >= 0; i--) {
         if (
           messages[i].role === "user" &&
-          messages[i].content.startsWith(`/${customCommand.name}`)
+          stripImages(messages[i].content).startsWith(`/${customCommand.name}`)
         ) {
           messages[i] = { ...messages[i], content: promptUserInput };
           break;
         }
       }
 
       for await (const chunk of llm.streamChat(messages)) {
-        yield chunk.content;
+        yield stripImages(chunk.content);
       }
     },
   };

core/commands/slash/cmd.ts

@@ -1,19 +1,40 @@
 import { SlashCommand } from "../..";
+import { streamLines } from "../../diff/util";
 import { removeQuotesAndEscapes } from "../../util";
 
 const GenerateTerminalCommand: SlashCommand = {
   name: "cmd",
   description: "Generate a shell command",
   run: async function* ({ ide, llm, input }) {
-    let cmd =
-      await llm.complete(`The user has made a request to run a shell command. Their description of what it should do is:
+    const gen =
+      llm.streamComplete(`The user has made a request to run a shell command. Their description of what it should do is:
 
 "${input}"
 
-Please write a shell command that will do what the user requested. Your output should consist of only the command itself, without any explanation or example output. Do not use any newlines. Only output the command that when inserted into the terminal will do precisely what was requested.`);
-    cmd = removeQuotesAndEscapes(cmd.trim())
-      .replace("\n", "")
-      .replace("\r", "");
+Please write a shell command that will do what the user requested. Your output should consist of only the command itself, without any explanation or example output. Do not use any newlines. Only output the command that when inserted into the terminal will do precisely what was requested. Here is the command:`);
+
+    const lines = streamLines(gen);
+    let cmd = "";
+    for await (const line of lines) {
+      console.log(line);
+      if (line.startsWith("```") && line.endsWith("```")) {
+        cmd = line.split(" ").slice(1).join(" ").slice(0, -3);
+        break;
+      }
+
+      if (
+        line.startsWith(">") ||
+        line.startsWith("``") ||
+        line.startsWith("\\begin{") ||
+        line.trim() === ""
+      ) {
+        continue;
+      }
+
+      cmd = removeQuotesAndEscapes(line.trim());
+      break;
+    }
+
     await ide.runCommand(cmd);
     yield `Generated shell command: ${cmd}`;
   },

core/commands/slash/commit.ts

@@ -1,4 +1,5 @@
 import { SlashCommand } from "../..";
+import { stripImages } from "../../llm/countTokens";
 
 const CommitMessageCommand: SlashCommand = {
   name: "commit",
@@ -9,7 +10,7 @@ const CommitMessageCommand: SlashCommand = {
     for await (const chunk of llm.streamChat([
       { role: "user", content: prompt },
     ])) {
-      yield chunk.content;
+      yield stripImages(chunk.content);
     }
   },
 };

core/commands/slash/draftIssue.ts

@@ -1,4 +1,5 @@
 import { ChatMessage, SlashCommand } from "../..";
+import { stripImages } from "../../llm/countTokens";
 import { removeQuotesAndEscapes } from "../../util";
 
 const PROMPT = (
@@ -28,12 +29,13 @@ const DraftIssueCommand: SlashCommand = {
       yield "This command requires a repository URL to be set in the config file.";
       return;
     }
-    const title = await llm.complete(
+    let title = await llm.complete(
       `Generate a title for the GitHub issue requested in this user input: '${input}'. Use no more than 20 words and output nothing other than the title. Do not surround it with quotes. The title is: `,
       { maxTokens: 20 }
     );
 
-    yield removeQuotesAndEscapes(title) + "\n\n";
+    title = removeQuotesAndEscapes(title.trim()) + "\n\n";
+    yield title;
 
     let body = "";
     const messages: ChatMessage[] = [
@@ -43,7 +45,7 @@ const DraftIssueCommand: SlashCommand = {
 
     for await (const chunk of llm.streamChat(messages)) {
       body += chunk.content;
-      yield chunk.content;
+      yield stripImages(chunk.content);
     }
 
     const url = `${params.repositoryUrl}/issues/new?title=${encodeURIComponent(

core/commands/slash/edit.ts

@@ -1,4 +1,5 @@
 import { ContextItemWithId, ILLM, SlashCommand } from "../..";
+import { stripImages } from "../../llm/countTokens";
 import { dedentAndGetCommonWhitespace, renderPromptTemplate } from "../../util";
 import {
   RangeInFileWithContents,
@@ -205,10 +206,15 @@ const EditSlashCommand: SlashCommand = {
   name: "edit",
   description: "Edit selected code",
   run: async function* ({ ide, llm, input, history, contextItems, params }) {
-    const contextItemToEdit = contextItems.find(
+    let contextItemToEdit = contextItems.find(
       (item: ContextItemWithId) =>
         item.editing && item.id.providerTitle === "code"
     );
+    if (!contextItemToEdit) {
+      contextItemToEdit = contextItems.find(
+        (item: ContextItemWithId) => item.id.providerTitle === "code"
+      );
+    }
 
     if (!contextItemToEdit) {
       yield "Select (highlight and press `cmd+shift+M` (MacOS) / `ctrl+shift+M` (Windows)) the code that you want to edit first";
@@ -457,7 +463,7 @@ const EditSlashCommand: SlashCommand = {
             4096
           ),
         })) {
-          yield chunk.content;
+          yield stripImages(chunk.content);
         }
       }
 

core/commands/slash/share.ts

@@ -1,4 +1,5 @@
 import { SlashCommand } from "../..";
+import { stripImages } from "../../llm/countTokens";
 
 const ShareSlashCommand: SlashCommand = {
   name: "share",
@@ -7,9 +8,9 @@ const ShareSlashCommand: SlashCommand = {
     let content = `This is a session transcript from [Continue](https://continue.dev) on ${new Date().toLocaleString()}.`;
 
     for (const msg of history) {
-      content += `\n\n## ${msg.role === "user" ? "User" : "Continue"}\n\n${
-        msg.content
-      }`;
+      content += `\n\n## ${
+        msg.role === "user" ? "User" : "Continue"
+      }\n\n${stripImages(msg.content)}`;
     }
 
     const continueDir = await ide.getContinueDir();

core/commands/slash/stackOverflow.ts

@@ -1,5 +1,5 @@
 import { ChatMessageRole, SlashCommand } from "../..";
-import { pruneStringFromBottom } from "../../llm/countTokens";
+import { pruneStringFromBottom, stripImages } from "../../llm/countTokens";
 
 const SERVER_URL = "https://proxy-server-l6vsfbzhba-uw.a.run.app";
 const PROMPT = (
@@ -114,7 +114,7 @@ const StackOverflowSlashCommand: SlashCommand = {
       })),
       { role: "user", content: PROMPT(input) },
     ])) {
-      yield chunk.content;
+      yield stripImages(chunk.content);
     }
   },
 };

core/commands/slash/verticalEdit.ts

@@ -1,6 +1,7 @@
 import { ContextItemWithId, DiffLine, ILLM, SlashCommand } from "../..";
 import { streamDiff } from "../../diff/streamDiff";
 import { LineStream, streamLines } from "../../diff/util";
+import { stripImages } from "../../llm/countTokens";
 import { gptEditPrompt } from "../../llm/templates/edit";
 import {
   dedentAndGetCommonWhitespace,
@@ -81,7 +82,7 @@ function constructPrompt(
   });
   return typeof rendered === "string"
     ? rendered
-    : rendered[rendered.length - 1].content;
+    : stripImages(rendered[rendered.length - 1].content);
 }
 
 async function* addIndentation(

core/config/default.ts

@@ -8,9 +8,9 @@ const defaultConfig: SerializedContinueConfig = {
       model: "gpt-4",
     },
     {
-      title: "GPT-3.5-Turbo (Free Trial)",
+      title: "GPT-4 Vision (Free Trial)",
       provider: "free-trial",
-      model: "gpt-3.5-turbo",
+      model: "gpt-4-vision-preview",
     },
     {
       title: "Phind CodeLlama (Free Trial)",

core/config/types.ts

@@ -13,6 +13,11 @@ declare global {
     index: number; // Index of the chunk in the document at filepath
   }
   
+  export interface IndexingProgressUpdate {
+    progress: number;
+    desc: string;
+  }
+  
   export interface LLMReturnValue {
     prompt: string;
     completion: string;
@@ -61,30 +66,49 @@ declare global {
     ): Promise<ChatMessage>;
   
     countTokens(text: string): number;
+  
+    supportsImages(): boolean;
   }
   
+  export type ContextProviderType = "normal" | "query" | "submenu";
+  
   export interface ContextProviderDescription {
     title: string;
     displayTitle: string;
     description: string;
-    dynamic: boolean;
-    requiresQuery: boolean;
+    type: ContextProviderType;
   }
   
-  interface ContextProviderExtras {
+  export interface ContextProviderExtras {
     fullInput: string;
-    embeddingsProvider?: EmbeddingsProvider;
+    embeddingsProvider: EmbeddingsProvider;
     llm: ILLM;
+    ide: IDE;
+    selectedCode: RangeInFile[];
+  }
+  
+  export interface LoadSubmenuItemsArgs {
+    ide: IDE;
   }
   
   export interface CustomContextProvider {
     title: string;
     displayTitle?: string;
     description?: string;
+    type?: ContextProviderType;
     getContextItems(
       query: string,
       extras: ContextProviderExtras
     ): Promise<ContextItem[]>;
+    loadSubmenuItems?: (
+      args: LoadSubmenuItemsArgs
+    ) => Promise<ContextSubmenuItem[]>;
+  }
+  
+  export interface ContextSubmenuItem {
+    id: string;
+    title: string;
+    description: string;
   }
   
   export interface IContextProvider {
@@ -94,6 +118,8 @@ declare global {
       query: string,
       extras: ContextProviderExtras
     ): Promise<ContextItem[]>;
+  
+    loadSubmenuItems(args: LoadSubmenuItemsArgs): Promise<ContextSubmenuItem[]>;
   }
   
   export interface PersistedSessionInfo {
@@ -140,9 +166,17 @@ declare global {
   
   export type ChatMessageRole = "user" | "assistant" | "system";
   
+  export interface MessagePart {
+    type: "text" | "imageUrl";
+    text?: string;
+    imageUrl?: { url: string };
+  }
+  
+  export type MessageContent = string | MessagePart[];
+  
   export interface ChatMessage {
     role: ChatMessageRole;
-    content: string;
+    content: MessageContent;
   }
   
   export interface ContextItemId {
@@ -280,6 +314,7 @@ declare global {
       diffLine: DiffLine
     ): Promise<void>;
     getOpenFiles(): Promise<string[]>;
+    getPinnedFiles(): Promise<string[]>;
     getSearchResults(query: string): Promise<string>;
     subprocess(command: string): Promise<[string, string]>;
     getProblems(filepath?: string | undefined): Promise<Problem[]>;
@@ -552,7 +587,7 @@ declare global {
     userToken?: string;
     embeddingsProvider: EmbeddingsProvider;
   }
-    
+      
 }
 
 export {};

core/context/index.ts

@@ -2,7 +2,9 @@ import {
   ContextItem,
   ContextProviderDescription,
   ContextProviderExtras,
+  ContextSubmenuItem,
   IContextProvider,
+  LoadSubmenuItemsArgs,
 } from "..";
 export abstract class BaseContextProvider implements IContextProvider {
   options: { [key: string]: any };
@@ -23,5 +25,9 @@ export abstract class BaseContextProvider implements IContextProvider {
     extras: ContextProviderExtras
   ): Promise<ContextItem[]>;
 
-  abstract load(): Promise<void>;
+  async loadSubmenuItems(
+    args: LoadSubmenuItemsArgs
+  ): Promise<ContextSubmenuItem[]> {
+    return [];
+  }
 }