Merge branch 'main' into tomasz/unify-snippets

.changes/extensions/vscode/0.8.58.md

@@ -0,0 +1,25 @@
+## 0.8.58 - 2024-11-22
+### Added
+* OpenAI predicted outputs support
+* Improve codebase retrieval with BM25
+* Support for Grok from xAI
+* Chat enhancements including sticking input to bottom
+* New UI for cmd+I in sidebar
+* Support for Nebius LLM provider
+* Support for Ask Sage LLM provider
+* Improved reference for config.json
+* New @web context provider
+* Updates for llama3.2
+* .continuerules file to set system prompt
+* .prompt files v2
+* Dedicated UI for docs indexing
+* Clickable code symbols in chat
+* Use clipboard content as autocomplete context
+### Changed
+* Improved @docs crawler
+* Many improvements to make autocomplete more eager
+* Near complete type definition retrieval for TypeScript autocomplete
+* Remove footer from chat sidebar
+### Fixed
+* Brought back the Apply button for all code blocks
+* Automatically update codebase index on removed files

core/autocomplete/context/ImportDefinitionsService.ts

@@ -1,5 +1,4 @@
-import { IDE } from "../..";
-import { RangeInFileWithContents } from "../../commands/util";
+import { IDE, RangeInFileWithContents } from "../..";
 import { PrecalculatedLruCache } from "../../util/LruCache";
 import {
   getFullLanguageName,

core/autocomplete/context/ranking/index.ts

@@ -1,8 +1,7 @@
-import { RangeInFileWithContents } from "../../../commands/util.js";
-import { Range } from "../../../index.js";
-import { countTokens } from "../../../llm/countTokens.js";
-import { AutocompleteSnippetDeprecated } from "../../types.js";
-import { HelperVars } from "../../util/HelperVars.js";
+import { RangeInFileWithContents } from "../../../";
+import { countTokens } from "../../../llm/countTokens";
+import { AutocompleteSnippetDeprecated } from "../../types";
+import { HelperVars } from "../../util/HelperVars";
 
 const rx = /[\s.,\/#!$%\^&\*;:{}=\-_`~()\[\]]/g;
 export function getSymbolsForSnippet(snippet: string): Set<string> {

core/autocomplete/filtering/streamTransforms/StreamTransformPipeline.ts

@@ -6,12 +6,12 @@ import { stopAtStopTokens } from "./charStream";
 import {
   avoidEmptyComments,
   avoidPathLine,
-  noDoubleNewlineAfterClosingBracket,
   showWhateverWeHaveAtXMs,
   skipPrefixes,
   stopAtLines,
   stopAtRepeatingLines,
   stopAtSimilarLine,
+  stopNCharsAfterClosingBracket,
   streamWithNewLines,
 } from "./lineStream";
 
@@ -48,7 +48,7 @@ export class StreamTransformPipeline {
     );
     lineGenerator = avoidPathLine(lineGenerator, helper.lang.singleLineComment);
     lineGenerator = skipPrefixes(lineGenerator);
-    lineGenerator = noDoubleNewlineAfterClosingBracket(lineGenerator);
+    lineGenerator = stopNCharsAfterClosingBracket(lineGenerator);
 
     for (const lineFilter of helper.lang.lineFilters ?? []) {
       lineGenerator = lineFilter({ lines: lineGenerator, fullStop });

core/autocomplete/filtering/streamTransforms/lineStream.ts

@@ -83,6 +83,7 @@ export const LINES_TO_REMOVE_BEFORE_START = [
   "<COMPLETION>",
   "[CODE]",
   "<START EDITING HERE>",
+  "{{FILL_HERE}}"
 ];
 
 export const ENGLISH_START_PHRASES = [
@@ -547,32 +548,21 @@ export async function* showWhateverWeHaveAtXMs(
   }
 }
 
-export async function* noDoubleNewlineAfterClosingBracket(
+export async function* stopNCharsAfterClosingBracket(
   lines: LineStream,
+  n: number = 20,
 ): LineStream {
   const bracketTypeCounts = new Map<string, number>();
+  let charsToStopAt: number | null = null;
 
   for await (const line of lines) {
-    if (line.trim() === "") {
-      // Double newline detected
-      // Check if any bracket counts are negative
-      let hasNegativeCount = false;
-      for (const count of bracketTypeCounts.values()) {
-        if (count < 0) {
-          hasNegativeCount = true;
-          break;
-        }
-      }
-      if (hasNegativeCount) {
-        // Stop the generator if we've closed brackets we didn't open
-        return;
-      }
-    }
+    let outputLine = "";
+    let i = 0;
 
-    yield line;
+    while (i < line.length) {
+      const char = line[i];
 
-    // Update bracket counts
-    for (const char of line) {
+      // Update bracket counts
       if (BRACKETS[char]) {
         // It's an opening bracket
         const count = bracketTypeCounts.get(char) || 0;
@@ -581,8 +571,32 @@ export async function* noDoubleNewlineAfterClosingBracket(
         // It's a closing bracket
         const openingBracket = BRACKETS_REVERSE[char];
         const count = bracketTypeCounts.get(openingBracket) || 0;
-        bracketTypeCounts.set(openingBracket, count - 1);
+        const newCount = count - 1;
+        bracketTypeCounts.set(openingBracket, newCount);
+
+        if (newCount < 0 && charsToStopAt === null) {
+          // Unmatched closing bracket detected
+          charsToStopAt = n;
+        }
       }
+
+      // Add the character to the output line
+      outputLine += char;
+
+      // If we've started counting down, decrement the remaining characters
+      if (charsToStopAt !== null) {
+        charsToStopAt -= 1;
+        if (charsToStopAt <= 0) {
+          // Yield the output line up to this point and stop the generator
+          yield outputLine;
+          return;
+        }
+      }
+
+      i += 1;
     }
+
+    // Yield whatever we've accumulated for this line
+    yield outputLine;
   }
 }