diff --git a/.changes/extensions/intellij/0.0.72.md b/.changes/extensions/intellij/0.0.72.md new file mode 100644 index 0000000000..2e8962e431 --- /dev/null +++ b/.changes/extensions/intellij/0.0.72.md @@ -0,0 +1,7 @@ +## 0.0.72 - 2024-10-04 +### Added +* Listen for changes to Intellij settings without requiring window reload +### Changed +* Updated tutorial file +### Fixed +* Fix ability to load config.ts in JetBrains IDEs diff --git a/.changes/extensions/vscode/0.8.55.md b/.changes/extensions/vscode/0.8.55.md new file mode 100644 index 0000000000..4800ad9076 --- /dev/null +++ b/.changes/extensions/vscode/0.8.55.md @@ -0,0 +1,9 @@ +## 0.8.55 - 2024-10-25 +### Added +* Web context provider +* Cerebras inference provider +* Automatic descriptions for previous chats +* Discord context provider +* Improved full screen UI +### Changed +* Easier way to accept/reject/re-prompt after cmd/ctrl+I diff --git a/.changes/extensions/vscode/0.8.56.md b/.changes/extensions/vscode/0.8.56.md new file mode 100644 index 0000000000..d1c81fd344 --- /dev/null +++ b/.changes/extensions/vscode/0.8.56.md @@ -0,0 +1,4 @@ +## 0.8.56 - 2024-11-08 +### Added +* New Edit mode in sidebar (cmd/ctrl+I) +* Significantly faster and more accurate docs crawler by default diff --git a/.changes/extensions/vscode/0.8.58.md b/.changes/extensions/vscode/0.8.58.md new file mode 100644 index 0000000000..98c845b973 --- /dev/null +++ b/.changes/extensions/vscode/0.8.58.md @@ -0,0 +1,25 @@ +## 0.8.58 - 2024-11-22 +### Added +* OpenAI predicted outputs support +* Improve codebase retrieval with BM25 +* Support for Grok from xAI +* Chat enhancements including sticking input to bottom +* New UI for cmd+I in sidebar +* Support for Nebius LLM provider +* Support for Ask Sage LLM provider +* Improved reference for config.json +* New @web context provider +* Updates for llama3.2 +* .continuerules file to set system prompt +* .prompt files v2 +* Dedicated UI for docs indexing +* Clickable code symbols in chat +* Use clipboard content as autocomplete context +### Changed +* Improved @docs crawler +* Many improvements to make autocomplete more eager +* Near complete type definition retrieval for TypeScript autocomplete +* Remove footer from chat sidebar +### Fixed +* Brought back the Apply button for all code blocks +* Automatically update codebase index on removed files diff --git a/.changes/extensions/vscode/0.8.59.md b/.changes/extensions/vscode/0.8.59.md new file mode 100644 index 0000000000..9f6dfbd04d --- /dev/null +++ b/.changes/extensions/vscode/0.8.59.md @@ -0,0 +1,3 @@ +## 0.8.59 - 2024-11-25 +### Fixed +* Hotfix for Ollama onboarding diff --git a/.changes/extensions/vscode/0.8.62.md b/.changes/extensions/vscode/0.8.62.md new file mode 100644 index 0000000000..c258fd2859 --- /dev/null +++ b/.changes/extensions/vscode/0.8.62.md @@ -0,0 +1,6 @@ +## 0.8.62 - 2024-12-10 +### Added +* Tool use +* Support for Model Context Protocol +### Fixed +* hotfix: context providers no longer hidden when not in edit mode diff --git a/.changes/unreleased/Changed-20240923-160251.yaml b/.changes/unreleased/Changed-20240923-160251.yaml deleted file mode 100644 index 5cef25babf..0000000000 --- a/.changes/unreleased/Changed-20240923-160251.yaml +++ /dev/null @@ -1,4 +0,0 @@ -project: extensions/intellij -kind: Changed -body: Updated tutorial file -time: 2024-09-23T16:02:51.918152-07:00 diff --git a/.changes/unreleased/Fixed-20240929-095440.yaml b/.changes/unreleased/Fixed-20240929-095440.yaml deleted file mode 100644 index d31fac222b..0000000000 --- a/.changes/unreleased/Fixed-20240929-095440.yaml +++ /dev/null @@ -1,4 +0,0 @@ -project: extensions/intellij -kind: Fixed -body: Fix ability to load config.ts in JetBrains IDEs -time: 2024-09-29T09:54:40.284968-07:00 diff --git a/.changes/unreleased/Fixed-20241007-183820.yaml b/.changes/unreleased/Fixed-20241007-183820.yaml new file mode 100644 index 0000000000..f756e6f90e --- /dev/null +++ b/.changes/unreleased/Fixed-20241007-183820.yaml @@ -0,0 +1,5 @@ +project: extensions/intellij +kind: Fixed +body: Off-screen rendering to solve white flash on load and lack of changing cursor + type +time: 2024-10-07T18:38:20.733658-07:00 diff --git a/.changes/unreleased/Fixed-20241108-114322.yaml b/.changes/unreleased/Fixed-20241108-114322.yaml new file mode 100644 index 0000000000..e896eac4d4 --- /dev/null +++ b/.changes/unreleased/Fixed-20241108-114322.yaml @@ -0,0 +1,4 @@ +project: extensions/intellij +kind: Fixed +body: OSR-related fixes for non-Mac users +time: 2024-11-08T11:43:22.623155-08:00 diff --git a/.changes/unreleased/Fixed-20241108-114707.yaml b/.changes/unreleased/Fixed-20241108-114707.yaml new file mode 100644 index 0000000000..95c70c8449 --- /dev/null +++ b/.changes/unreleased/Fixed-20241108-114707.yaml @@ -0,0 +1,4 @@ +project: extensions/intellij +kind: Fixed +body: Fixes for inline edit in JetBrains +time: 2024-11-08T11:47:07.953304-08:00 diff --git a/.continue/prompts/core-unit-test.prompt b/.continue/prompts/core-unit-test.prompt new file mode 100644 index 0000000000..42ecef7bba --- /dev/null +++ b/.continue/prompts/core-unit-test.prompt @@ -0,0 +1,41 @@ +name: Write Core Unit Test +description: Generate unit tests for core utilities +--- +Write jest tests for the provided code. +Use jest version ^29 (e.g. jest 29.7.0) + +Use best practices. Be clear and consise. +Aim for 100% code coverage where reasonable. +Multiple tests can be written, split up tests for best clarity and readability. +Only use typescript, and if the file/code is not typescript, warn the user. +IMPORTANT Use ESM to import modules, do NOT use `require` anywhere +Tests are to be described in an adjacent file with a path identical except for a `.test.ts` rather than a `.ts` file extension +Use double quotes (or backticks if needed) for strings + +The code being tested is used in IDE extensions, and it: +- accesses code workspaces through the IDE ("workspace directories") +- persists extension-related data to the the local machine of the user ("global directory"), and +- uses configuration via a `ConfigHandler`, which is stored in either the global directory (default) or accesed via a remote "control plane" using the `ControlPlaneClient` + +Jest testing setup includes +- @core/test/jest.global-setup.ts initializes a temporary global directory, which is where files that store persisted extension data live. +- @core/test/testDir.ts provides utils for creating and working with the temporary workspace directory. Use `setUpTestDir` and `tearDownTestDir` explicitly in tests that work with workspace files +- @core/test/jest.setup-after-env.ts gives tests access to node and jest globals + +@core/test/fixtures.ts provides fixtures that should be used in tests to emulate extension behavior +- import `testIde` for IDE/workspace operations +- import `testConfigHandler` for any ConfigHandler needs +- import `testControlPlaneClient` for control plane operations +- import `ideSettingsPromise` for any IdeSettings needs +- import `testLLM` for any ILLM/BaseLLM needs. Set the `completion` property to the desired completion, e.g. `testLLM.completion = "Desired completion";` + +Do NOT write tests for any files in `core/test`, only use them as helpers for testing other files. If no other files are provided, warn the user and write no tests. + +IMPORTANT: Do NOT mock the fixtures above other than using `jest.spyOn`. DO mock 3rd party modules, etc. when sensible. +Instead, generate actual mock files and data for operations +Pure mocks should only be used to emulate specific network responses/error or hard-to-duplicate errors, or to prevent long-duration tests + +Additional types can be imported from @core/index.d.ts +If any needed types, functions, constants, or classes are still not found, warn the user and do not generate tests. + +Write the comment "// Generated by continue" at the top of the generated code/file (not the filepath) \ No newline at end of file diff --git a/.continueignore b/.continueignore index 7e0d0c115f..79e924012d 100644 --- a/.continueignore +++ b/.continueignore @@ -4,4 +4,5 @@ docs/docs/languages .idea/ .vscode/ .archive/ -**/*.scm \ No newline at end of file +**/*.scm +**/*.diff \ No newline at end of file diff --git a/.eslintrc.shared.json b/.eslintrc.shared.json new file mode 100644 index 0000000000..f4b9c4c783 --- /dev/null +++ b/.eslintrc.shared.json @@ -0,0 +1,36 @@ +{ + "parser": "@typescript-eslint/parser", + "parserOptions": { + "ecmaVersion": 6, + "sourceType": "module" + }, + "plugins": ["@typescript-eslint", "import"], + "rules": { + "@typescript-eslint/semi": "warn", + "curly": "warn", + "eqeqeq": "warn", + "no-throw-literal": "warn", + "semi": "off", + "import/order": [ + "warn", + { + "groups": [ + "builtin", + "external", + "internal", + "parent", + "sibling", + "index", + "object", + "type" + ], + "alphabetize": { + "order": "asc", + "caseInsensitive": true + }, + "newlines-between": "always" + } + ] + }, + "ignorePatterns": ["out", "dist", "**/*.d.ts"] +} diff --git a/.github/auto_assign.yml b/.github/auto_assign.yml new file mode 100644 index 0000000000..ed05df6f9d --- /dev/null +++ b/.github/auto_assign.yml @@ -0,0 +1,10 @@ +# Reference: https://github.com/marketplace/actions/auto-assign-action +addAssignees: true + +assignees: + - sestinj + - Patrick-Erichsen + - tomasz-stefaniak + - RomneyDa + +numberOfAssignees: 2 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index f9a4b0151a..4e7157e670 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -4,8 +4,7 @@ ## Checklist -- [ ] The base branch of this PR is `dev`, rather than `main` -- [ ] The relevant docs, if any, have been updated or created +- [] The relevant docs, if any, have been updated or created ## Screenshots @@ -14,3 +13,4 @@ ## Testing [ For new or modified features, provide step-by-step testing instructions to validate the intended behavior of the change. ] +[ For new UI features, ensure that the changes look good across viewport widths, light/dark theme, etc ] diff --git a/.github/workflows/README.md b/.github/workflows/README.md deleted file mode 100644 index ff1b7bd22c..0000000000 --- a/.github/workflows/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Deployment Process - -## preview - -When merging to `preview`: - -- the VS Code extension along with Rust extension is built and uploaded as an artifact -- all of the artifacts are downloaded and pushed to the store/registry all at once, as full releases. -- the version is bumped and this change is commited to preview -- in the future, the JetBrains extension will be built and uploaded to the marketplace here - -## main - -When merging to `main`: - -- the VS Code extension along with Rust extension is built and uploaded as an artifact -- all of the artifacts are downloaded and pushed to the store/registry all at once, as full releases. -- the version is bumped and this change is commited to main -- in the future, the JetBrains extension will be built and uploaded to the marketplace here diff --git a/.github/workflows/auto-assign.yaml b/.github/workflows/auto-assign-issue.yaml similarity index 54% rename from .github/workflows/auto-assign.yaml rename to .github/workflows/auto-assign-issue.yaml index 1dd452dae9..dc5388ef32 100644 --- a/.github/workflows/auto-assign.yaml +++ b/.github/workflows/auto-assign-issue.yaml @@ -3,8 +3,6 @@ name: Issue assignment on: issues: types: [opened] - pull_request: - types: [opened] jobs: auto-assign: @@ -16,5 +14,11 @@ jobs: uses: pozil/auto-assign-issue@v2 with: repo-token: ${{ secrets.CI_GITHUB_TOKEN }} - assignees: sestinj,Patrick-Erichsen + assignees: sestinj,Patrick-Erichsen,tomasz-stefaniak,RomneyDa numOfAssignee: 1 + - name: "Add default labels" + uses: actions-ecosystem/action-add-labels@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + labels: | + "needs-triage" diff --git a/.github/workflows/auto-assign-pr.yaml b/.github/workflows/auto-assign-pr.yaml new file mode 100644 index 0000000000..4fd701a154 --- /dev/null +++ b/.github/workflows/auto-assign-pr.yaml @@ -0,0 +1,10 @@ +name: PR assignment +on: + pull_request: + types: [opened, ready_for_review] + +jobs: + add-reviews: + runs-on: ubuntu-latest + steps: + - uses: kentaro-m/auto-assign-action@v2.0.0 diff --git a/.github/workflows/dev_pr.yaml b/.github/workflows/dev_pr.yaml deleted file mode 100644 index d909a279ab..0000000000 --- a/.github/workflows/dev_pr.yaml +++ /dev/null @@ -1,92 +0,0 @@ -name: Dev PR checks - -on: - pull_request: - branches: - - dev - -jobs: - tsc-check: - runs-on: ubuntu-latest - - steps: - # 1. Check-out repository - - name: Check-out repository - uses: actions/checkout@v4 - - # 2. Install npm dependencies - - name: Use Node.js from .nvmrc - uses: actions/setup-node@v4 - with: - node-version-file: ".nvmrc" - - - name: Cache extension node_modules - uses: actions/cache@v3 - with: - path: extensions/vscode/node_modules - key: ${{ runner.os }}-node-${{ hashFiles('extensions/vscode/package-lock.json') }} - - - name: Cache core node_modules - uses: actions/cache@v3 - with: - path: core/node_modules - key: ${{ runner.os }}-node-${{ hashFiles('core/package-lock.json') }} - - - name: Cache gui node_modules - uses: actions/cache@v3 - with: - path: gui/node_modules - key: ${{ runner.os }}-node-${{ hashFiles('gui/package-lock.json') }} - - - name: Cache binary node_modules - uses: actions/cache@v3 - with: - path: binary/node_modules - key: ${{ runner.os }}-node-${{ hashFiles('binary/package-lock.json') }} - - - name: extensions/vscode install - run: | - cd extensions/vscode - npm ci - env: - # https://github.com/microsoft/vscode-ripgrep/issues/9#issuecomment-643965333 - GITHUB_TOKEN: ${{ secrets.CI_GITHUB_TOKEN }} - - - name: core install - run: | - cd core - npm ci - - - name: gui install - run: | - cd gui - npm ci - - - name: binary install - run: | - cd binary - npm ci - - - name: extensions/vscode checks - run: | - cd extensions/vscode - npx tsc --noEmit - - - name: core checks - run: | - cd core - npm ci - npx tsc --noEmit - npm run lint - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - - - name: gui checks - run: | - cd gui - npx tsc --noEmit - - - name: binary checks - run: | - cd binary - npx tsc --noEmit diff --git a/.github/workflows/jetbrains-release.yaml b/.github/workflows/jetbrains-release.yaml index cf3cc4ae19..b8c473604f 100644 --- a/.github/workflows/jetbrains-release.yaml +++ b/.github/workflows/jetbrains-release.yaml @@ -97,7 +97,6 @@ jobs: with: node-version-file: ".nvmrc" - # Cache node_modules - name: Cache core node_modules uses: actions/cache@v3 with: @@ -320,7 +319,6 @@ jobs: chmod +x continue-binary chmod +x build/Release/node_sqlite3.node chmod +x index.node - chmod +x esbuild if: ${{ matrix.platform }} != 'win32' # Run tests for binary @@ -408,7 +406,7 @@ jobs: # Run Qodana inspections - name: Qodana - Code Inspection - uses: JetBrains/qodana-action@v2024.1.5 + uses: JetBrains/qodana-action@v2024.3.2 with: cache-default-branch-only: true diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 459774b74f..ae0359027d 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -49,11 +49,11 @@ jobs: platform: alpine arch: x64 npm_config_arch: x64 - - os: macos-12 # should migrate this to the newer x64 version of macos-14 + - os: macos-13 platform: darwin arch: x64 npm_config_arch: x64 - - os: macos-12 # same here, especially + - os: macos-13 platform: darwin arch: arm64 npm_config_arch: arm64 @@ -213,8 +213,8 @@ jobs: git config --local user.email "action@github.com" git config --local user.name "GitHub Action" - - name: Pull latest changes - run: git pull origin main + - name: Pull latest changes from release + run: git fetch origin ${{ github.ref }} && git checkout ${{ github.ref }} # 1. Download the artifacts - uses: actions/download-artifact@v4 @@ -237,29 +237,3 @@ jobs: run: | cd extensions/vscode npx ovsx publish -p ${{ secrets.VSX_REGISTRY_TOKEN }} --packagePath ../../vsix-artifacts/*.vsix - - # 4. Update the package.json version and push changes - # - name: Update version in package.json - # run: | - # cd extensions/vscode - # npm version patch - - # - name: Commit changes - # run: | - # git config --local user.email "action@github.com" - # git config --local user.name "GitHub Action" - # git commit -am "💚 Update package.json version [skip ci]" - - # - name: Push changes - # uses: ad-m/github-push-action@master - # with: - # github_token: ${{ secrets.GITHUB_TOKEN }} - # branch: ${{ github.ref }} - - # 5 Send to Discord Webhook - - name: Discord Commits - uses: Sniddl/discord-commits@1.7 - with: - webhook: ${{ secrets.DISCORD_WEBHOOK }} - template: "avatar-with-link" - include-extras: true diff --git a/.github/workflows/pr_checks.yaml b/.github/workflows/pr_checks.yaml new file mode 100644 index 0000000000..cfe67c7195 --- /dev/null +++ b/.github/workflows/pr_checks.yaml @@ -0,0 +1,254 @@ +name: PR checks + +on: + pull_request: + branches: + - main + paths: + - "extensions/vscode/**" + - "core/**" + - "gui/**" + - ".github/workflows/**" + +jobs: + install-root: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".nvmrc" + + - uses: actions/cache@v4 + id: root-cache + with: + path: node_modules + key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }} + + - name: Install root dependencies + if: steps.root-cache.outputs.cache-hit != 'true' + run: npm ci + + core-checks: + needs: install-root + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".nvmrc" + + - uses: actions/cache@v4 + id: root-cache + with: + path: node_modules + key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }} + + - uses: actions/cache@v4 + id: core-cache + with: + path: core/node_modules + key: ${{ runner.os }}-core-node-modules-${{ hashFiles('core/package-lock.json') }} + + - name: Install core dependencies + if: steps.core-cache.outputs.cache-hit != 'true' + run: | + cd core + npm ci + + - name: Type check and lint + run: | + cd core + npx tsc --noEmit + npm run lint + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + + gui-checks: + needs: [install-root, core-checks] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".nvmrc" + + - uses: actions/cache@v4 + with: + path: node_modules + key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }} + + - uses: actions/cache@v4 + with: + path: core/node_modules + key: ${{ runner.os }}-core-node-modules-${{ hashFiles('core/package-lock.json') }} + + - uses: actions/cache@v4 + id: gui-cache + with: + path: gui/node_modules + key: ${{ runner.os }}-gui-node-modules-${{ hashFiles('gui/package-lock.json') }} + + - name: Install gui dependencies + if: steps.gui-cache.outputs.cache-hit != 'true' + run: | + cd gui + npm ci + + - name: Type check + run: | + cd gui + npx tsc --noEmit + + binary-checks: + needs: [install-root, core-checks] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".nvmrc" + + - uses: actions/cache@v4 + with: + path: node_modules + key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }} + + - uses: actions/cache@v4 + with: + path: core/node_modules + key: ${{ runner.os }}-core-node-modules-${{ hashFiles('core/package-lock.json') }} + + - uses: actions/cache@v4 + id: binary-cache + with: + path: binary/node_modules + key: ${{ runner.os }}-binary-node-modules-${{ hashFiles('binary/package-lock.json') }} + + - name: Install binary dependencies + if: steps.binary-cache.outputs.cache-hit != 'true' + run: | + cd binary + npm ci + + - name: Type check + run: | + cd binary + npx tsc --noEmit + + vscode-checks: + needs: [install-root, core-checks] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".nvmrc" + + - uses: actions/cache@v4 + with: + path: node_modules + key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }} + + - uses: actions/cache@v4 + with: + path: core/node_modules + key: ${{ runner.os }}-core-node-modules-${{ hashFiles('core/package-lock.json') }} + + - uses: actions/cache@v4 + id: vscode-cache + with: + path: extensions/vscode/node_modules + key: ${{ runner.os }}-vscode-node-modules-${{ hashFiles('extensions/vscode/package-lock.json') }} + + - name: Install vscode dependencies + if: steps.vscode-cache.outputs.cache-hit != 'true' + run: | + cd extensions/vscode + npm ci + env: + GITHUB_TOKEN: ${{ secrets.CI_GITHUB_TOKEN }} + + - name: Type check and lint + run: | + cd extensions/vscode + npx tsc --noEmit + npm run lint + + core-tests: + needs: [core-checks] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".nvmrc" + + - uses: actions/cache@v4 + with: + path: core/node_modules + key: ${{ runner.os }}-core-node-modules-${{ hashFiles('core/package-lock.json') }} + + - name: Run core tests + run: | + cd core + npm test + + vscode-tests: + needs: [vscode-checks, core-checks] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".nvmrc" + + - uses: actions/cache@v4 + with: + path: extensions/vscode/node_modules + key: ${{ runner.os }}-vscode-node-modules-${{ hashFiles('extensions/vscode/package-lock.json') }} + + - uses: actions/cache@v4 + with: + path: core/node_modules + key: ${{ runner.os }}-core-node-modules-${{ hashFiles('core/package-lock.json') }} + + - name: Install Xvfb for Linux and run e2e tests + run: | + sudo apt-get install -y xvfb # Install Xvfb + Xvfb :99 & # Start Xvfb + export DISPLAY=:99 # Export the display number to the environment + cd extensions/vscode + npm run package + npm run e2e:ci + + gui-tests: + needs: [gui-checks, core-checks] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".nvmrc" + + - uses: actions/cache@v4 + id: gui-cache + with: + path: gui/node_modules + key: ${{ runner.os }}-gui-node-modules-${{ hashFiles('gui/package-lock.json') }} + + - uses: actions/cache@v4 + with: + path: core/node_modules + key: ${{ runner.os }}-core-node-modules-${{ hashFiles('core/package-lock.json') }} + + - name: Install GUI dependencies + if: steps.gui-cache.outputs.cache-hit != 'true' + run: cd gui && npm ci + env: + GITHUB_TOKEN: ${{ secrets.CI_GITHUB_TOKEN }} + + - name: Run gui tests + run: | + cd gui + npm test diff --git a/.github/workflows/preview.yaml b/.github/workflows/preview.yaml index fbc69b64f9..bd5f4b7acb 100644 --- a/.github/workflows/preview.yaml +++ b/.github/workflows/preview.yaml @@ -49,11 +49,11 @@ jobs: platform: alpine arch: x64 npm_config_arch: x64 - - os: macos-12 # should migrate this to the newer x64 version of macos-14 + - os: macos-13 platform: darwin arch: x64 npm_config_arch: x64 - - os: macos-12 # same here, especially + - os: macos-13 platform: darwin arch: arm64 npm_config_arch: arm64 @@ -210,7 +210,7 @@ jobs: git config --local user.name "GitHub Action" - name: Pull latest changes - run: git pull origin dev + run: git pull origin main # 1. Download the artifacts - uses: actions/download-artifact@v4 diff --git a/.github/workflows/submit-github-dependency-graph.yml b/.github/workflows/submit-github-dependency-graph.yml new file mode 100644 index 0000000000..c705484a9c --- /dev/null +++ b/.github/workflows/submit-github-dependency-graph.yml @@ -0,0 +1,25 @@ +name: Submit Gradle Dependency Graph For Dependabot + +on: + push: + branches: ['main'] + +permissions: + contents: write + +jobs: + dependency-submission: + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + - name: Setup Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + - name: Generate and submit dependency graph + uses: gradle/actions/dependency-submission@v4 + with: + # The gradle project is not in the root of the repository. + build-root-directory: extensions/intellij diff --git a/.github/workflows/ts-check.yaml b/.github/workflows/ts-check.yaml deleted file mode 100644 index f73016fa5d..0000000000 --- a/.github/workflows/ts-check.yaml +++ /dev/null @@ -1,89 +0,0 @@ -name: TypeScript Check - -on: - pull_request: - branches: - - main - - preview - -jobs: - tsc-check: - runs-on: ubuntu-latest - - steps: - # 1. Check-out repository - - name: Check-out repository - uses: actions/checkout@v4 - - # 2. Install npm dependencies - - name: Use Node.js from .nvmrc - uses: actions/setup-node@v4 - with: - node-version-file: ".nvmrc" - - - name: Cache extension node_modules - uses: actions/cache@v3 - with: - path: extensions/vscode/node_modules - key: ${{ runner.os }}-node-${{ hashFiles('extensions/vscode/package-lock.json') }} - - - name: Cache core node_modules - uses: actions/cache@v3 - with: - path: core/node_modules - key: ${{ runner.os }}-node-${{ hashFiles('core/package-lock.json') }} - - - name: Cache gui node_modules - uses: actions/cache@v3 - with: - path: gui/node_modules - key: ${{ runner.os }}-node-${{ hashFiles('gui/package-lock.json') }} - - - name: Cache binary node_modules - uses: actions/cache@v3 - with: - path: binary/node_modules - key: ${{ runner.os }}-node-${{ hashFiles('binary/package-lock.json') }} - - - name: Install extension Dependencies - run: | - cd extensions/vscode - npm ci - env: - # https://github.com/microsoft/vscode-ripgrep/issues/9#issuecomment-643965333 - GITHUB_TOKEN: ${{ secrets.CI_GITHUB_TOKEN }} - - - name: Install gui Dependencies - run: | - cd gui - npm ci - - - name: Install Core Dependencies - run: | - cd core - npm ci - - - name: Install Binary Dependencies - run: | - cd binary - npm ci - - - name: tsc core - run: | - cd core - npx tsc --noEmit - - - name: tsc extensions/vscode - run: | - cd extensions/vscode - npx tsc --noEmit - - - name: tsc binary - run: | - cd binary - npx tsc --noEmit - - - name: tsc gui - run: | - cd gui - npx tsc --noEmit diff --git a/.gitignore b/.gitignore index ba5199b567..af20fc0c18 100644 --- a/.gitignore +++ b/.gitignore @@ -135,38 +135,36 @@ cached_embeddings.pkl .ruff_cache codeql -**/.continue .DS_Store -.continue .test .tiktoken_cache -# IntelliJ Plugin +# IntelliJ Plugin **/**/.gradle **/**/.qodana **/**/build -**/.idea/**/* -!**/.idea/.name -!**/.idea/compiler.xml -!**/.idea/gradle.xml -!**/.idea/kotlinc.xml -!**/.idea/misc.xml -!**/.idea/vcs.xml -!**/.idea/jarRepositories.xml -core/.idea - continue_server.build continue_server.dist Icon Icon? -.continue - - -.prompts/ +.continuerc.json .aider* notes.md + +manual-testing-sandbox/.idea/** +extensions/intellij/.idea/** + +**/.idea/workspace.xml +**/.idea/usage.statistics.xml +**/.idea/shelf/ + +extensions/intellij/bin +extensions/.epico-pilot-debug/ +extensions/.continue-debug/ + +*.vsix diff --git a/.idea/.name b/.idea/.name index c0d78d1d57..389f6bc71e 100644 --- a/.idea/.name +++ b/.idea/.name @@ -1 +1 @@ -Continue IntelliJ Extension \ No newline at end of file +continue-intellij-extension \ No newline at end of file diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml new file mode 100644 index 0000000000..a98e3ae475 --- /dev/null +++ b/.idea/codeStyles/Project.xml @@ -0,0 +1,63 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000000..79ee123c2b --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/.idea/continue.iml b/.idea/continue.iml new file mode 100644 index 0000000000..d6ebd48059 --- /dev/null +++ b/.idea/continue.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/extensions/intellij/.idea/compiler.xml b/.idea/git_toolbox_blame.xml similarity index 50% rename from extensions/intellij/.idea/compiler.xml rename to .idea/git_toolbox_blame.xml index b589d56e9f..7dc124965d 100644 --- a/extensions/intellij/.idea/compiler.xml +++ b/.idea/git_toolbox_blame.xml @@ -1,6 +1,6 @@ - - + + \ No newline at end of file diff --git a/.idea/gradle.xml b/.idea/gradle.xml index 6733405f98..1d937028b0 100644 --- a/.idea/gradle.xml +++ b/.idea/gradle.xml @@ -5,7 +5,6 @@ - diff --git a/.idea/misc.xml b/.idea/misc.xml index efa79f7dfb..2b4b067217 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -4,7 +4,7 @@ - + diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000000..c8a08c311d --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/modules/com.github.continuedev.continueintellijextension.continue-intellij-extension.iml b/.idea/modules/com.github.continuedev.continueintellijextension.continue-intellij-extension.iml new file mode 100644 index 0000000000..713a88bca4 --- /dev/null +++ b/.idea/modules/com.github.continuedev.continueintellijextension.continue-intellij-extension.iml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/modules/com.github.continuedev.continueintellijextension.continue-intellij-extension.main.iml b/.idea/modules/com.github.continuedev.continueintellijextension.continue-intellij-extension.main.iml new file mode 100644 index 0000000000..d2841a905d --- /dev/null +++ b/.idea/modules/com.github.continuedev.continueintellijextension.continue-intellij-extension.main.iml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/modules/com.github.continuedev.continueintellijextension.continue-intellij-extension.test.iml b/.idea/modules/com.github.continuedev.continueintellijextension.continue-intellij-extension.test.iml new file mode 100644 index 0000000000..dbca5d9917 --- /dev/null +++ b/.idea/modules/com.github.continuedev.continueintellijextension.continue-intellij-extension.test.iml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/prettier.xml b/.idea/prettier.xml new file mode 100644 index 0000000000..b0c1c68fbb --- /dev/null +++ b/.idea/prettier.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/scopes/Continue.xml b/.idea/scopes/Continue.xml new file mode 100644 index 0000000000..3b28a76b1f --- /dev/null +++ b/.idea/scopes/Continue.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/.instructions b/.instructions new file mode 100644 index 0000000000..bad5d96891 --- /dev/null +++ b/.instructions @@ -0,0 +1 @@ +Give concise responses. diff --git a/.prettierignore b/.prettierignore index db326191f8..15619929df 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,4 +1,17 @@ +binary/bin +binary/build +binary/out +binary/tmp +core/.continue-test +docs/.docusaurus +extensions/.continue-debug extensions/vscode/continue_rc_schema.json +extensions/vscode/.vscode-test +extensions/vscode/bin +extensions/vscode/build +extensions/vscode/out +gui/dist **/.continueignore CHANGELOG.md **/continue_tutorial.py +**/node_modules \ No newline at end of file diff --git a/.prettierrc b/.prettierrc index d3be6d22ea..7b5484dd7e 100644 --- a/.prettierrc +++ b/.prettierrc @@ -1,5 +1,9 @@ { "tabWidth": 2, "useTabs": false, - "trailingComma": "all" + "trailingComma": "all", + "semi": true, + "singleQuote": false, + "bracketSpacing": true, + "plugins": ["prettier-plugin-tailwindcss"] } diff --git a/.prompts/test.prompt b/.prompts/test.prompt deleted file mode 100644 index e2f5d86623..0000000000 --- a/.prompts/test.prompt +++ /dev/null @@ -1,19 +0,0 @@ -temperature: 0.5 -maxTokens: 4096 -name: jest -description: Write Jest unit tests ---- - -You are an expert programmer - - -{{{ input }}} - -Write unit tests for the above selected code, following each of these instructions: -- Use `jest` -- Properly set up and tear down -- Include important edge cases -- The tests should be complete and sophisticated -- Give the tests just as chat output, don't edit any file -- Don't explain how to set up `jest` -- Write a single code block, making sure to label with the language being used (e.g. "```typscript") \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 9c9e607c7e..faa14c6a40 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -20,7 +20,10 @@ "outFiles": ["${workspaceFolder}/extensions/vscode/out/extension.js"], "preLaunchTask": "vscode-extension:build", "env": { - // "CONTROL_PLANE_ENV": "local" + // "CONTINUE_GLOBAL_DIR": "${workspaceFolder}/extensions/.epico-pilot-debug" + // "staging" for the preview deployment "CONTINUE_GLOBAL_DIR": "${workspaceFolder}/extensions/.epico-pilot-debug" + // "local" for entirely local development of control plane/proxy + // "CONTROL_PLANE_ENV": "staging" } }, { @@ -29,7 +32,6 @@ "name": "Core Binary", "skipFiles": ["/**"], "program": "${workspaceFolder}/binary/out/index.js", - // "preLaunchTask": "binary:esbuild", "outFiles": ["${workspaceFolder}/binary/out/**/*.js"], "sourceMaps": true, "smartStep": true, @@ -37,7 +39,7 @@ "cwd": "${workspaceFolder}/binary", "env": { "CONTINUE_DEVELOPMENT": "true", - "CONTINUE_GLOBAL_DIR": "${workspaceFolder}/binary/.epi-copilot" + "CONTINUE_GLOBAL_DIR": "${workspaceFolder}/binary/.epico-pilot" } }, { @@ -136,24 +138,5 @@ "NODE_OPTIONS": "--experimental-vm-modules" } } - // { - // "name": "[openai-adapters] Jest Test Debugger, Current Open File", - // "type": "node", - // "request": "launch", - // "runtimeArgs": [ - // "--inspect-brk", - // "${workspaceRoot}/packages/openai-adapters/node_modules/jest/bin/jest.js", - // "--runInBand", - // "--config", - // "${workspaceRoot}/packages/openai-adapters/jest.config.mjs", - // "${relativeFile}" - // ], - // "cwd": "${workspaceRoot}/packages/openai-adapters", - // "console": "integratedTerminal", - // "internalConsoleOptions": "neverOpen", - // "env": { - // "NODE_OPTIONS": "--experimental-vm-modules" - // } - // } ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index 41b3577650..22d8d32873 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -13,13 +13,29 @@ "args": ["-l"] } }, + "editor.defaultFormatter": "esbenp.prettier-vscode", "search.exclude": { - "**/binary": true, - "binary/": false, - "**/core/vendor/**": true, - "**/gui/dist": true + "**/package-lock.json": true, + "binary/bin/**": true, + "binary/build/**": true, + "binary/out/**": true, + "binary/tmp/**": true, + "core/edit/lazy/test-examples/**": true, + "core/llm/llamaTokenizer.js": true, + "core/llm/llamaTokenizer.mjs": true, + "core/vendor/**": true, + "docs/.docusaurus/**": true, + "extensions/intellij/build/**": true, + "extensions/vscode/bin/**": true, + "extensions/vscode/build/**": true, + "extensions/vscode/gui/**": true, + "extensions/vscode/models/all-MiniLM-L6-v2/**": true, + "extensions/vscode/out/**": true, + "extensions/vscode/tag-qry/**": true, + "extensions/vscode/textmate-syntaxes/**": true, + "extensions/vscode/tree-sitter/**": true, + "gui/dist/**": true + // "sync/**": true }, - "eslint.workingDirectories": [ - "./core" - ] + "eslint.workingDirectories": ["./core"] } diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 2e595561f2..e3250d80a9 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -1,16 +1,36 @@ { "version": "2.0.0", "tasks": [ + { + "label": "tsc:watch", + "type": "npm", + "script": "tsc:watch", + "isBackground": true, + "problemMatcher": { + "base": "$tsc-watch", + "background": { + "activeOnStart": true, + "beginsPattern": "Starting compilation in watch mode...", + "endsPattern": "Found 0 errors" + } + }, + "presentation": { + "revealProblems": "onProblem" + } + }, // Compile and bundle the extension { "label": "vscode-extension:build", "dependsOn": [ - // To detect compile errors - "vscode-extension:tsc", + // TSC type checking + "tsc:watch", // To build the React app that is used in the extension "vscode-extension:continue-ui:build", - // To bundle the code the same way we do for publishing + // // To bundle the code the same way we do for publishing "vscode-extension:esbuild", + // To ensure extension.js is regenerated even if + // vscode-extension:esbuild was already running in background + "vscode-extension:esbuild-notify", // Start the React app that is used in the extension "gui:dev" ], @@ -19,8 +39,16 @@ "isDefault": true } }, + { + "label": "vscode-extension:esbuild-notify", + "dependsOn": ["vscode-extension:esbuild"], + "type": "npm", + "script": "esbuild-notify", + "path": "extensions/vscode" + }, { "label": "vscode-extension:esbuild", + "dependsOn": ["vscode-extension:continue-ui:build"], "type": "npm", "script": "esbuild-watch", "path": "extensions/vscode", @@ -40,22 +68,11 @@ "background": { "activeOnStart": true, "beginsPattern": ">", - "endsPattern": ">" + "endsPattern": "VS Code Extension esbuild complete" } } ] }, - { - "label": "vscode-extension:tsc", - "type": "npm", - "script": "tsc", - "path": "extensions/vscode", - "problemMatcher": ["$tsc"], - "presentation": { - "revealProblems": "onProblem", - "clear": true - } - }, // Build the React app. It gets bundled into the extension as a file resource and has a seprate build step { "label": "vscode-extension:continue-ui:build", @@ -79,7 +96,7 @@ // Build the extension "vscode-extension:build", // To detect compile errors - this type checks both the extension and the tests - "vscode-extension:tsc", + "tsc:watch", "vscode-extension:tests:esbuild" ] }, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c070f0bc1b..c2d8bb49d1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,11 +15,11 @@ - [🧑💻 Contributing Code](#-contributing-code) - [Environment Setup](#environment-setup) - [Pre-requisites](#pre-requisites) - - [Fork the Continue Repository with All Branches](#fork-the-continue-repository-with-all-branches) + - [Fork the Continue Repository with All Branches](#fork-the-continue-repository) - [VS Code](#vs-code) - [Debugging](#debugging) - [JetBrains](#jetbrains) - - [Debugging](#debugging-1) + - [Our Git Workflow](#our-git-workflow) - [Formatting](#formatting) - [Writing Slash Commands](#writing-slash-commands) - [Writing Context Providers](#writing-context-providers) @@ -68,7 +68,7 @@ You can run the documentation server locally using either of the following metho #### Method 1: NPM Script -1. Open your terminal and navigate to the `docs` subdirectory of the project. The `docusaurus.config.js` file you'll see there is a sign you're in the right place. +1. Open your terminal and navigate to the `docs` subdirectory of the project. The `docusaurus.config.js` file you'll see there is a sign you're in the right place. 2. Run the following command to install the necessary dependencies for the documentation server: @@ -94,8 +94,6 @@ This will start a local server and you can see the documentation rendered in you ## 🧑💻 Contributing Code -> Please make PRs to the `dev` branch. We use this to first test changes in a pre-release version of the extension. - ### Environment Setup #### Pre-requisites @@ -106,15 +104,15 @@ You should have Node.js version 20.11.0 (LTS) or higher installed. You can get i nvm use ``` -#### Fork the Continue Repository with All Branches +#### Fork the Continue Repository -1. Go to the [Continue GitHub repository](https://github.com/continuedev/continue) and fork it to your GitHub account. **Ensure all branches are included in the fork**. +1. Go to the [Continue GitHub repository](https://github.com/continuedev/continue) and fork it to your GitHub account. 2. Clone your forked repository to your local machine. Use: `git clone https://github.com/YOUR_USERNAME/continue.git` -3. Navigate to the cloned directory and switch to the **dev** branch. Execute: `git checkout dev`, then create your feature/fix branch from there, like so: `git checkout -b 123-my-feature-branch` +3. Navigate to the cloned directory and make sure you are on the main branch. Create your feature/fix branch from there, like so: `git checkout -b 123-my-feature-branch` -4. When you're ready to submit your changes, send your pull request specifically to the **dev** branch. +4. Send your pull request to the main branch. #### VS Code @@ -141,37 +139,11 @@ Similarly, any changes to `core` or `extensions/vscode` will be automatically in #### JetBrains -Pre-requisite: You should use the Intellij IDE, which can be downloaded [here](https://www.jetbrains.com/idea/download). Either Ultimate or Community (free) will work. Continue is built with JDK version 17, as specified in `extensions/intellij/build.gradle.kts`. You should also ensure that you have the Gradle plugin installed. - -1. Clone the repository -2. Run `scripts/install-dependencies.sh` or `scripts/install-dependencies.ps1` on Windows. This will install and build all of the necessary dependencies. -3. To test the plugin, select the "Run Plugin" Gradle configuration and click the "Run" or "Debug" button as shown in this screenshot: - ![img](./media/IntelliJRunPluginScreenshot.png) -4. To package the extension, run `./gradlew build` (or `./gradlew.bat build` on Windows) from the `extensions/intellij` directory. This will generate a .zip file in `extensions/intellij/build/distributions` with the version defined in `extensions/intellij/gradle.properties`. -5. If you make changes, you may need to re-build before running the "Build Plugin" configuration - - a. If you change code from the `core` or `binary` directories, make sure to run `npm run build` from the `binary` directory to create a new binary. - - b. If you change code from the `gui` directory, make sure to run `npm run build` from the `gui` directory to create a new bundle. - - c. Any changes to the Kotlin coded in the `extensions/intellij` directory will be automatically included when you run "Build Plugin" - -##### Debugging - -Continue's JetBrains extension shares much of the code with the VS Code extension by utilizing shared code in the `core` directory and packaging it in a binary in the `binary` directory. The JetBrains extension (written in Kotlin) is then able to communicate over stdin/stdout in the [CoreMessenger.kt](./extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/CoreMessenger.kt) file. - -For the sake of rapid development, it is also possible to configure this communication to happen over local TCP sockets: - -1. In [CoreMessenger.kt](./extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/CoreMessenger.kt), change the `useTcp` variable to `true`. -2. Open a VS Code window (we recommend this for a preconfigured Typescript debugging experience) with the `continue` repository. Select the "Core Binary" debug configuration and press play. -3. Run the "Run Plugin" Gradle configuration. -4. You can now set breakpoints in any of the TypeScript files in VS Code. If you make changes to the code, restart the "Core Binary" debug configuration and reload the _Host IntelliJ_ window. - -If you make changes to Kotlin code, they can often be hot-reloaded with "Run -> Debugging Actions -> Reload Changed Classes". +See the [`CONTRIBUTING.md`](./extensions/intellij/CONTRIBUTING.md) for the JetBrains extension. ### Our Git Workflow -We keep two permanent branches: `main` and `dev`. All contributions should be made as pull requests to the `dev` branch. When we are ready to create a "pre-release" version, we create a tag on the `dev` branch, which automatically triggers the workflow in [preview.yaml](./.github/workflows/preview.yaml), which builds and releases a version of the VS Code extension. When a release has been sufficiently tested, we will merge its tag into the `main` branch. Creating a tag on the `main` branch will then trigger a similar workflow in [main.yaml](./.github/workflows/main.yaml), which will build and release a main release of the VS Code extension. Any hotfixes can be made by creating a feature branch from the tag for the release in question. +We keep a single permanent branch: `main`. When we are ready to create a "pre-release" version, we create a tag on the `main` branch titled `v0.9.x-vscode`, which automatically triggers the workflow in [preview.yaml](./.github/workflows/preview.yaml), which builds and releases a version of the VS Code extension. When a release has been sufficiently tested, we will create a new release titled `v0.8.x-vscode`, triggering a similar workflow in [main.yaml](./.github/workflows/main.yaml), which will build and release a main release of the VS Code extension. Any hotfixes can be made by creating a feature branch from the tag for the release in question. This workflow is well explained by http://releaseflow.org. ### Formatting @@ -226,10 +198,9 @@ Continue has support for more than a dozen different LLM "providers", making it While any model that works with a supported provider can be used with Continue, we keep a list of recommended models that can be automatically configured from the UI or `config.json`. The following files should be updated when adding a model: - [config_schema.json](./extensions/vscode/config_schema.json) - This is the JSON Schema definition that is used to validate `config.json`. You'll notice a number of rules defined in "definitions.ModelDescription.allOf". Here is where you write rules that can specify something like "for the provider 'anthropic', only models 'claude-2' and 'claude-instant-1' are allowed. Look through all of these rules and make sure that your model is included for providers that support it. -- [modelData.ts](./gui/src/util/modelData.ts) - This file defines that information that is shown in the model selection UI in the side bar. To add a new model: - 1. create a `ModelPackage` object, following the lead of the many examples near the top of the file - 2. add the `ModelPackage` to the `MODEL_INFO` array if you would like it to be displayed in the "Models" tab - 3. if you would like it to be displayed as an option under any of the providers, go to the `PROVIDER_INFO` object and add it to the `packages` array for each provider that you want it to be displayed under. If it is an OS model that should be valid for most providers offering OS models, you might just be able to add it to the `osModels` array as shorthand. +- [AddNewModel page](./gui/src/pages/AddNewModel) - This directory defines which model options are shown in the side bar model selection UI. To add a new model: + 1. Add a `ModelPackage` entry for the model into [configs/models.ts](./gui/src/pages/AddNewModel/configs/models.ts), following the lead of the many examples near the top of the file + 2. Add the model within its provider's array to [AddNewModel.tsx](./gui/src/pages/AddNewModel/AddNewModel.tsx) (add provider if needed) - [index.d.ts](./core/index.d.ts) - This file defines the TypeScript types used throughout Continue. You'll find a `ModelName` type. Be sure to add the name of your model to this. - LLM Providers: Since many providers use their own custom strings to identify models, you'll have to add the translation from Continue's model name (the one you added to `index.d.ts`) and the model string for each of these providers: [Ollama](./core/llm/llms/Ollama.ts), [Together](./core/llm/llms/Together.ts), and [Replicate](./core/llm/llms/Replicate.ts). You can find their full model lists here: [Ollama](https://ollama.ai/library), [Together](https://docs.together.ai/docs/inference-models), [Replicate](https://replicate.com/collections/streaming-language-models). - [Prompt Templates](./core/llm/index.ts) - In this file you'll find the `autodetectTemplateType` function. Make sure that for the model name you just added, this function returns the correct template type. This is assuming that the chat template for that model is already built in Continue. If not, you will have to add the template type and corresponding edit and chat templates. diff --git a/README.md b/README.md index 5972563251..9751d618dc 100644 --- a/README.md +++ b/README.md @@ -62,4 +62,4 @@ Check out the [contribution ideas board](https://github.com/orgs/continuedev/pro ## License -[Apache 2.0 © 2023 Continue Dev, Inc.](./LICENSE) +[Apache 2.0 © 2023-2024 Continue Dev, Inc.](./LICENSE) diff --git a/extensions/intellij/.idea/vcs.xml b/binary/.idea/vcs.xml similarity index 68% rename from extensions/intellij/.idea/vcs.xml rename to binary/.idea/vcs.xml index b2bdec2d71..6c0b863585 100644 --- a/extensions/intellij/.idea/vcs.xml +++ b/binary/.idea/vcs.xml @@ -1,6 +1,6 @@ - + \ No newline at end of file diff --git a/binary/build.js b/binary/build.js index 016afe9bd2..e537ac67e0 100644 --- a/binary/build.js +++ b/binary/build.js @@ -118,9 +118,6 @@ async function installNodeModuleInTempDirAndCopyToCurrent(packageName, toCopy) { } (async () => { - fs.mkdirSync("out/node_modules", { recursive: true }); - fs.mkdirSync("bin/node_modules", { recursive: true }); - console.log("[info] Downloading prebuilt lancedb..."); for (const target of targets) { if (targetToLanceDb[target]) { @@ -270,24 +267,6 @@ async function installNodeModuleInTempDirAndCopyToCurrent(packageName, toCopy) { fs.unlinkSync(`${targetDir}/build.tar.gz`); - // Download and unzip prebuilt esbuild binary for the target - console.log(`[info] Downloading esbuild for ${target}...`); - // Version is pinned to 0.19.11 in package.json to make sure that they match - execCmdSync( - `curl -o ${targetDir}/esbuild.tgz https://registry.npmjs.org/@esbuild/${target}/-/${target}-0.19.11.tgz`, - ); - execCmdSync(`tar -xzvf ${targetDir}/esbuild.tgz -C ${targetDir}`); - if (target.startsWith("win32")) { - fs.cpSync(`${targetDir}/package/esbuild.exe`, `${targetDir}/esbuild.exe`); - } else { - fs.cpSync(`${targetDir}/package/bin/esbuild`, `${targetDir}/esbuild`); - } - fs.rmSync(`${targetDir}/esbuild.tgz`); - fs.rmSync(`${targetDir}/package`, { - force: true, - recursive: true, - }); - // copy @lancedb to bin folders console.log("[info] Copying @lancedb files to bin"); fs.copyFileSync( @@ -295,9 +274,6 @@ async function installNodeModuleInTempDirAndCopyToCurrent(packageName, toCopy) { `${targetDir}/index.node`, ); } - // execCmdSync( - // `npx pkg out/index.js --target node18-darwin-arm64 --no-bytecode --public-packages "*" --public -o bin/pkg` - // ); const pathsToVerify = []; for (target of targets) { @@ -305,7 +281,6 @@ async function installNodeModuleInTempDirAndCopyToCurrent(packageName, toCopy) { const targetDir = `bin/${target}`; pathsToVerify.push( `${targetDir}/continue-binary${exe}`, - `${targetDir}/esbuild${exe}`, `${targetDir}/index.node`, // @lancedb "package.json", // Informs of where to look for node_sqlite3.node https://www.npmjs.com/package/bindings#:~:text=The%20searching%20for,file%20is%20found `${targetDir}/build/Release/node_sqlite3.node`, diff --git a/binary/package-lock.json b/binary/package-lock.json index 5f1410c1a7..6f7c394d1d 100644 --- a/binary/package-lock.json +++ b/binary/package-lock.json @@ -47,12 +47,14 @@ "@aws-sdk/client-bedrock-runtime": "^3.620.1", "@aws-sdk/client-sagemaker-runtime": "^3.621.0", "@aws-sdk/credential-providers": "^3.620.1", - "@continuedev/config-types": "^1.0.10", + "@continuedev/config-types": "^1.0.13", + "@continuedev/fetch": "^1.0.4", "@continuedev/llm-info": "^1.0.2", + "@continuedev/openai-adapters": "^1.0.10", + "@modelcontextprotocol/sdk": "^1.0.0", "@mozilla/readability": "^0.5.0", "@octokit/rest": "^20.0.2", "@typescript-eslint/eslint-plugin": "^7.8.0", - "@typescript-eslint/parser": "^7.8.0", "@xenova/transformers": "2.14.0", "adf-to-md": "^1.1.0", "async-mutex": "^0.5.0", @@ -60,11 +62,12 @@ "cheerio": "^1.0.0-rc.12", "commander": "^12.0.0", "comment-json": "^4.2.3", - "dbinfoz": "^0.11.0", + "dbinfoz": "^0.14.0", "diff": "^7.0.0", "dotenv": "^16.4.5", "fastest-levenshtein": "^1.0.16", "follow-redirects": "^1.15.5", + "google-auth-library": "^9.14.2", "handlebars": "^4.7.8", "http-proxy-agent": "^7.0.1", "https-proxy-agent": "^7.0.3", @@ -74,13 +77,15 @@ "jsdom": "^24.0.0", "launchdarkly-node-client-sdk": "^3.2.0", "llm-code-highlighter": "^0.0.14", + "lru-cache": "^11.0.2", "mac-ca": "^3.1.0", "node-fetch": "^3.3.2", "node-html-markdown": "^1.3.0", "ollama": "^0.4.6", "onnxruntime-node": "1.14.0", - "openai": "^4.20.1", + "openai": "^4.76.0", "p-limit": "^6.1.0", + "partial-json": "^0.1.7", "pg": "^8.11.3", "posthog-node": "^3.6.3", "puppeteer": "^22.4.0", @@ -92,11 +97,13 @@ "sqlite": "^5.1.1", "sqlite3": "^5.1.7", "system-ca": "^1.0.3", + "tar": "^7.4.3", "tree-sitter-wasms": "^0.1.11", "uuid": "^9.0.1", "vectordb": "^0.4.20", "web-tree-sitter": "^0.21.0", "win-ca": "^3.5.1", + "wink-nlp-utils": "^2.1.0", "workerpool": "^9.1.3", "yaml": "^2.4.2", "zod": "^3.23.8" @@ -115,12 +122,12 @@ "@types/node-fetch": "^2.6.11", "@types/pg": "^8.11.6", "@types/request": "^2.48.12", + "@types/tar": "^6.1.13", "@types/uuid": "^9.0.7", "@types/win-ca": "^3.5.4", "cross-env": "^7.0.3", "esbuild": "0.17.19", "eslint": "^8", - "eslint-plugin-import": "^2.29.1", "jest": "^29.7.0", "jest-environment-jsdom": "^29.7.0", "myers-diff": "^2.1.0", @@ -133,12 +140,12 @@ } }, "node_modules/@75lb/deep-merge": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@75lb/deep-merge/-/deep-merge-1.1.1.tgz", - "integrity": "sha512-xvgv6pkMGBA6GwdyJbNAnDmfAIR/DfWhrj9jgWh3TY7gRm3KO46x/GPjRg6wJ0nOepwqrNxFfojebh0Df4h4Tw==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@75lb/deep-merge/-/deep-merge-1.1.2.tgz", + "integrity": "sha512-08K9ou5VNbheZFxM5tDWoqjA3ImC50DiuuJ2tj1yEPRfkp8lLLg6XAaJ4On+a0yAXor/8ay5gHnAIshRM44Kpw==", "peer": true, "dependencies": { - "lodash.assignwith": "^4.2.0", + "lodash": "^4.17.21", "typical": "^7.1.1" }, "engines": { @@ -2747,12 +2754,12 @@ } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -3543,9 +3550,9 @@ "optional": true }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, "dependencies": { "to-regex-range": "^5.0.1" @@ -5035,10 +5042,10 @@ "node": ">=8" } }, - "node_modules/lodash.assignwith": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/lodash.assignwith/-/lodash.assignwith-4.2.0.tgz", - "integrity": "sha512-ZznplvbvtjK2gMvnQ1BR/zqPFZmS6jbK4p+6Up4xcRYA7yMIwxHCfbTcrYxXKzzqLsQ05eJPVznEW3tuwV7k1g==", + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "peer": true }, "node_modules/lodash.camelcase": { diff --git a/binary/package.json b/binary/package.json index cb14c50f77..62a28b7418 100644 --- a/binary/package.json +++ b/binary/package.json @@ -23,7 +23,6 @@ "scripts": { "test": "jest", "build": "node build.js", - "build:old": "ncc build src/index.ts -o out && pkg .", "build:dev": "tsc", "esbuild": "node build.js --esbuild-only" }, diff --git a/binary/src/IpcIde.ts b/binary/src/IpcIde.ts index ee2649c365..5c9ad1afe1 100644 --- a/binary/src/IpcIde.ts +++ b/binary/src/IpcIde.ts @@ -1,5 +1,5 @@ import { TODO } from "core/util"; -import { MessageIde } from "core/util/messageIde"; +import { MessageIde } from "core/protocol/messenger/messageIde"; export class IpcIde extends MessageIde { constructor(messenger: TODO) { diff --git a/binary/src/IpcMessenger.ts b/binary/src/IpcMessenger.ts index 341372d8e8..ec6f160c75 100644 --- a/binary/src/IpcMessenger.ts +++ b/binary/src/IpcMessenger.ts @@ -1,5 +1,5 @@ import { IProtocol } from "core/protocol/index.js"; -import { IMessenger, type Message } from "core/util/messenger"; +import { IMessenger, type Message } from "core/protocol/messenger"; import { ChildProcessWithoutNullStreams } from "node:child_process"; import * as fs from "node:fs"; import net from "node:net"; diff --git a/binary/src/TcpMessenger.ts b/binary/src/TcpMessenger.ts index 9dc506d15c..ecd5d0d70e 100644 --- a/binary/src/TcpMessenger.ts +++ b/binary/src/TcpMessenger.ts @@ -1,5 +1,5 @@ import { IProtocol } from "core/protocol"; -import { IMessenger, Message } from "core/util/messenger"; +import { IMessenger, Message } from "core/protocol/messenger"; import net from "net"; import { v4 as uuidv4 } from "uuid"; diff --git a/binary/src/index.ts b/binary/src/index.ts index 04be5de5ad..df0a8520f6 100644 --- a/binary/src/index.ts +++ b/binary/src/index.ts @@ -2,7 +2,7 @@ process.env.IS_BINARY = "true"; import { Command } from "commander"; import { Core } from "core/core"; import { FromCoreProtocol, ToCoreProtocol } from "core/protocol"; -import { IMessenger } from "core/util/messenger"; +import { IMessenger } from "core/protocol/messenger"; import { getCoreLogsPath, getPromptLogsPath } from "core/util/paths"; import fs from "node:fs"; import { IpcIde } from "./IpcIde"; diff --git a/binary/test/binary.test.ts b/binary/test/binary.test.ts index 3554aed27b..f84b23a2d7 100644 --- a/binary/test/binary.test.ts +++ b/binary/test/binary.test.ts @@ -2,8 +2,7 @@ import { SerializedContinueConfig } from "core"; // import Mock from "core/llm/llms/Mock.js"; import { FromIdeProtocol, ToIdeProtocol } from "core/protocol/index.js"; import FileSystemIde from "core/util/filesystem"; -import { IMessenger } from "core/util/messenger"; -import { ReverseMessageIde } from "core/util/reverseMessageIde"; +import { IMessenger } from "core/protocol/messenger"; import fs from "fs"; import { ChildProcessWithoutNullStreams, @@ -133,7 +132,7 @@ describe("Test Suite", () => { fs.mkdirSync(testDir); } const ide = new FileSystemIde(testDir); - const reverseIde = new ReverseMessageIde(messenger.on.bind(messenger), ide); + // const reverseIde = new ReverseMessageIde(messenger.on.bind(messenger), ide); // Wait for core to set itself up await new Promise((resolve) => setTimeout(resolve, 1000)); diff --git a/binary/tsconfig.json b/binary/tsconfig.json index 215a5aee7f..9b3259730a 100644 --- a/binary/tsconfig.json +++ b/binary/tsconfig.json @@ -7,7 +7,8 @@ "sourceMap": true, "rootDirs": ["src", "../core"], "allowJs": true, - "strict": true /* enable all strict type-checking options */, + "skipLibCheck": true, + "strict": true /* enable all strict tsc:watching options */, /* Additional Checks */ // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ diff --git a/core/.eslintrc.json b/core/.eslintrc.json index 04097f4db6..ce1bc2575f 100644 --- a/core/.eslintrc.json +++ b/core/.eslintrc.json @@ -1,21 +1,12 @@ { "root": true, - "parser": "@typescript-eslint/parser", + "extends": ["../.eslintrc.shared.json"], "parserOptions": { - "ecmaVersion": 6, - "sourceType": "module", "project": "./tsconfig.json" }, - "plugins": ["@typescript-eslint", "import"], "rules": { "quotes": ["warn", "double", {}], "@typescript-eslint/naming-convention": "off", - "@typescript-eslint/semi": "warn", - "curly": "warn", - "eqeqeq": "warn", - "no-throw-literal": "warn", - "semi": "off", "@typescript-eslint/no-floating-promises": "warn" - }, - "ignorePatterns": ["out", "dist", "**/*.d.ts"] + } } diff --git a/core/.gitignore b/core/.gitignore index 7d8f1fa533..b34a983de4 100644 --- a/core/.gitignore +++ b/core/.gitignore @@ -3,6 +3,5 @@ target **/.DS_Store npm-debug.log* .env -.continue-test -testDir +test/.continue-test coverage \ No newline at end of file diff --git a/core/autocomplete/CompletionProvider.ts b/core/autocomplete/CompletionProvider.ts new file mode 100644 index 0000000000..e37d8227c1 --- /dev/null +++ b/core/autocomplete/CompletionProvider.ts @@ -0,0 +1,282 @@ +import { ConfigHandler } from "../config/ConfigHandler.js"; +import { TRIAL_FIM_MODEL } from "../config/onboarding.js"; +import { IDE, ILLM } from "../index.js"; +import OpenAI from "../llm/llms/OpenAI.js"; +import { DEFAULT_AUTOCOMPLETE_OPTS } from "../util/parameters.js"; +import { PosthogFeatureFlag, Telemetry } from "../util/posthog.js"; + +import { shouldCompleteMultiline } from "./classification/shouldCompleteMultiline.js"; +import { ContextRetrievalService } from "./context/ContextRetrievalService.js"; +// @prettier-ignore + +import { BracketMatchingService } from "./filtering/BracketMatchingService.js"; +import { CompletionStreamer } from "./generation/CompletionStreamer.js"; +import { postprocessCompletion } from "./postprocessing/index.js"; +import { shouldPrefilter } from "./prefiltering/index.js"; +import { getAllSnippets } from "./snippets/index.js"; +import { renderPrompt } from "./templating/index.js"; +import { GetLspDefinitionsFunction } from "./types.js"; +import { AutocompleteDebouncer } from "./util/AutocompleteDebouncer.js"; +import { AutocompleteLoggingService } from "./util/AutocompleteLoggingService.js"; +import AutocompleteLruCache from "./util/AutocompleteLruCache.js"; +import { HelperVars } from "./util/HelperVars.js"; +import { AutocompleteInput, AutocompleteOutcome } from "./util/types.js"; + +const autocompleteCache = AutocompleteLruCache.get(); + +// Errors that can be expected on occasion even during normal functioning should not be shown. +// Not worth disrupting the user to tell them that a single autocomplete request didn't go through +const ERRORS_TO_IGNORE = [ + // From Ollama + "unexpected server status", + "operation was aborted", +]; + +export class CompletionProvider { + private autocompleteCache = AutocompleteLruCache.get(); + public errorsShown: Set = new Set(); + private bracketMatchingService = new BracketMatchingService(); + private debouncer = new AutocompleteDebouncer(); + private completionStreamer: CompletionStreamer; + private loggingService = new AutocompleteLoggingService(); + private contextRetrievalService: ContextRetrievalService; + + constructor( + private readonly configHandler: ConfigHandler, + private readonly ide: IDE, + private readonly _injectedGetLlm: () => Promise, + private readonly _onError: (e: any) => void, + private readonly getDefinitionsFromLsp: GetLspDefinitionsFunction, + ) { + this.completionStreamer = new CompletionStreamer(this.onError.bind(this)); + this.contextRetrievalService = new ContextRetrievalService(this.ide); + } + + private async _prepareLlm(): Promise { + const llm = await this._injectedGetLlm(); + + if (!llm) { + return undefined; + } + + // Temporary fix for JetBrains autocomplete bug as described in https://github.com/continuedev/continue/pull/3022 + if (llm.model === undefined && llm.completionOptions?.model !== undefined) { + llm.model = llm.completionOptions.model; + } + + // Ignore empty API keys for Mistral since we currently write + // a template provider without one during onboarding + if (llm.providerName === "mistral" && llm.apiKey === "") { + return undefined; + } + + // Set temperature (but don't override) + if (llm.completionOptions.temperature === undefined) { + const value = await Telemetry.getValueForFeatureFlag( + PosthogFeatureFlag.AutocompleteTemperature, + ); + + llm.completionOptions.temperature = value ?? 0.01; + } + + if (llm instanceof OpenAI) { + llm.useLegacyCompletionsEndpoint = true; + } else if ( + llm.providerName === "free-trial" && + llm.model !== TRIAL_FIM_MODEL + ) { + llm.model = TRIAL_FIM_MODEL; + } + + return llm; + } + + private onError(e: any) { + if ( + ERRORS_TO_IGNORE.some((err) => + typeof e === "string" ? e.includes(err) : e?.message?.includes(err), + ) + ) { + return; + } + + console.warn("Error generating autocompletion: ", e); + if (!this.errorsShown.has(e.message)) { + this.errorsShown.add(e.message); + this._onError(e); + } + } + + public cancel() { + this.loggingService.cancel(); + } + + public accept(completionId: string) { + const outcome = this.loggingService.accept(completionId); + if (!outcome) { + return; + } + this.bracketMatchingService.handleAcceptedCompletion( + outcome.completion, + outcome.filepath, + ); + } + + public markDisplayed(completionId: string, outcome: AutocompleteOutcome) { + this.loggingService.markDisplayed(completionId, outcome); + } + + private async _getAutocompleteOptions() { + const config = await this.configHandler.loadConfig(); + const options = { + ...DEFAULT_AUTOCOMPLETE_OPTS, + ...config.tabAutocompleteOptions, + }; + return options; + } + + public async provideInlineCompletionItems( + input: AutocompleteInput, + token: AbortSignal | undefined, + ): Promise { + try { + const startTime = Date.now(); + const options = await this._getAutocompleteOptions(); + + // Debounce + if (await this.debouncer.delayAndShouldDebounce(options.debounceDelay)) { + return undefined; + } + + const llm = await this._prepareLlm(); + if (!llm) { + return undefined; + } + + const helper = await HelperVars.create( + input, + options, + llm.model, + this.ide, + ); + + if (await shouldPrefilter(helper, this.ide)) { + return undefined; + } + + // Create abort signal if not given + if (!token) { + const controller = this.loggingService.createAbortController( + input.completionId, + ); + token = controller.signal; + } + + const [snippetPayload, workspaceDirs] = await Promise.all([ + getAllSnippets({ + helper, + ide: this.ide, + getDefinitionsFromLsp: this.getDefinitionsFromLsp, + contextRetrievalService: this.contextRetrievalService, + }), + this.ide.getWorkspaceDirs(), + ]); + + const { prompt, prefix, suffix, completionOptions } = renderPrompt({ + snippetPayload, + workspaceDirs, + helper, + }); + + // Completion + let completion: string | undefined = ""; + + const cache = await autocompleteCache; + const cachedCompletion = helper.options.useCache + ? await cache.get(helper.prunedPrefix) + : undefined; + let cacheHit = false; + if (cachedCompletion) { + // Cache + cacheHit = true; + completion = cachedCompletion; + } else { + const multiline = + !helper.options.transform || shouldCompleteMultiline(helper); + + const completionStream = + this.completionStreamer.streamCompletionWithFilters( + token, + llm, + prefix, + suffix, + prompt, + multiline, + completionOptions, + helper, + ); + + for await (const update of completionStream) { + completion += update; + } + + // Don't postprocess if aborted + if (token.aborted) { + return undefined; + } + + const processedCompletion = helper.options.transform + ? postprocessCompletion({ + completion, + prefix: helper.prunedPrefix, + suffix: helper.prunedSuffix, + llm, + }) + : completion; + + completion = processedCompletion; + } + + if (!completion) { + return undefined; + } + + const outcome: AutocompleteOutcome = { + time: Date.now() - startTime, + completion, + prefix, + suffix, + prompt, + modelProvider: llm.providerName, + modelName: llm.model, + completionOptions, + cacheHit, + filepath: helper.filepath, + completionId: helper.input.completionId, + gitRepo: await this.ide.getRepoName(helper.filepath), + uniqueId: await this.ide.getUniqueId(), + timestamp: Date.now(), + ...helper.options, + }; + + ////////// + + // Save to cache + if (!outcome.cacheHit && helper.options.useCache) { + (await this.autocompleteCache).put(outcome.prefix, outcome.completion); + } + + // When using the JetBrains extension, Mark as displayed + const ideType = (await this.ide.getIdeInfo()).ideType; + if (ideType === "jetbrains") { + this.markDisplayed(input.completionId, outcome); + } + + return outcome; + } catch (e: any) { + this.onError(e); + } finally { + this.loggingService.deleteAbortController(input.completionId); + } + } +} diff --git a/core/autocomplete/NearbyDefinitionsService.ts b/core/autocomplete/NearbyDefinitionsService.ts deleted file mode 100644 index 53178cba7a..0000000000 --- a/core/autocomplete/NearbyDefinitionsService.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { IDE, Location } from "../index.js"; -import { LANGUAGES } from "./languages.js"; -import { getSymbolsForSnippet } from "./ranking.js"; - -interface FileInfo { - filepath: string; -} - -export class NearbyDefinitionsService { - static N = 10; - - constructor(private readonly ide: IDE) {} - - async getDefinitionsForLine(filepath: string, line: number) { - const lineContent = await this.ide.readRangeInFile(filepath, { - start: { - line, - character: 0, - }, - end: { - line: line + 1, - character: 0, - }, - }); - - // Remove keywords - const lang = LANGUAGES[filepath.split(".").slice(-1)[0]]; - const symbols = Array.from(getSymbolsForSnippet(lineContent)) - .filter((s) => s.length > 0) - .filter((s) => !(lang && lang?.stopWords?.includes(s))); - - return Promise.all( - symbols.map((s) => { - const character = lineContent.indexOf(s); - const pos: Location = { - filepath, - position: { - line, - character, - }, - }; - }), - ); - } -} diff --git a/core/autocomplete/README.md b/core/autocomplete/README.md index e19509bd8f..627ca12060 100644 --- a/core/autocomplete/README.md +++ b/core/autocomplete/README.md @@ -7,7 +7,7 @@ Continue now provides support for tab autocomplete in [VS Code](https://marketpl We recommend setting up tab-autocomplete with a local Ollama instance. To do this, first download the latest version of Ollama from [here](https://ollama.ai). Then, run the following command to download our recommended model: ```bash -ollama run starcoder:3b +ollama run qwen2.5-coder:1.5b ``` Once it has been downloaded, you should begin to see completions in VS Code. @@ -17,9 +17,9 @@ Once it has been downloaded, you should begin to see completions in VS Code. You can also set up tab-autocomplete with a local LM Studio instance by following these steps: 1. Download the latest version of LM Studio from [here](https://lmstudio.ai/) -2. Download a model (e.g. search for `second-state/StarCoder2-3B-GGUF` and choose one of the options there) +2. Download a model (e.g. search for `Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF` and choose one of the options there) 3. Go to the server section (button is on the left), select your model from the dropdown at the top, and click "Start Server" -4. Go to the "My Models" section (button is on the left), find your selected model, and copy the name the path (example: `second-state/StarCoder2-3B-GGUF/starcoder2-3b-Q8_0.gguf`); this will be used as the "model" attribute in Continue +4. Go to the "My Models" section (button is on the left), find your selected model, and copy the name the path (example: `Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF/qwen2.5-coder-1.5b-instruct-q4_k_m.gguf`); this will be used as the "model" attribute in Continue 5. Go to Continue and modify the configurations for a [custom model](#setting-up-a-custom-model) 6. Set the "provider" to `lmstudio` and the "model" to the path copied earlier @@ -28,8 +28,8 @@ Example: ```json title="config.json" { "tabAutocompleteModel": { - "title": "Starcoder2 3b", - "model": "second-state/StarCoder2-3B-GGUF/starcoder2-3b-Q8_0.gguf", + "title": "Qwen2.5-Coder 1.5b", + "model": "Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF", "provider": "lmstudio", }, ... @@ -69,11 +69,9 @@ If you aren't yet familiar with the available options, you can learn more in our ### What model should I use? -If you are running the model locally, we recommend `starcoder:3b`. +If you are running the model locally, we recommend `qwen2.5-coder:1.5b`. -If you find it to be too slow, you should try `deepseek-coder:1.3b-base`. - -If you have a bit more compute, or are running a model in the cloud, you can upgrade to `deepseek-coder:6.7b-base`. +If you have a bit more compute, or are running a model in the cloud, you can upgrade to `qwen2.5-coder:7b`. Regardless of what you are willing to spend, we do not recommend using GPT or Claude for autocomplete. Learn why [below](#i-want-better-completions-should-i-use-gpt-4). @@ -83,13 +81,12 @@ The following can be configured in `config.json`: ### `tabAutocompleteModel` -This is just another object like the ones in the `"models"` array of `config.json`. You can choose and configure any model you would like, but we strongly suggest using a small model made for tab-autocomplete, such as `deepseek-1b`, `starcoder-1b`, or `starcoder-3b`. +This is just another object like the ones in the `"models"` array of `config.json`. You can choose and configure any model you would like, but we strongly suggest using a small model made for tab-autocomplete, such as `deepseek-1b`, `qwen2.5-coder:1.5b`, or `starcoder-3b`. ### `tabAutocompleteOptions` This object allows you to customize the behavior of tab-autocomplete. The available options are: -- `useCopyBuffer`: Determines whether the copy buffer will be considered when constructing the prompt. (Boolean) - `useFileSuffix`: Determines whether to use the file suffix in the prompt. (Boolean) - `maxPromptTokens`: The maximum number of prompt tokens to use. A smaller number will yield faster completions, but less context. (Number) - `debounceDelay`: The delay in milliseconds before triggering autocomplete after a keystroke. (Number) @@ -105,11 +102,10 @@ This object allows you to customize the behavior of tab-autocomplete. The availa "tabAutocompleteModel": { "title": "Tab Autocomplete Model", "provider": "ollama", - "model": "starcoder:3b", + "model": "qwen2.5-coder:1.5b", "apiBase": "https://" }, "tabAutocompleteOptions": { - "useCopyBuffer": false, "maxPromptTokens": 400, "prefixPercentage": 0.5 } @@ -120,7 +116,7 @@ This object allows you to customize the behavior of tab-autocomplete. The availa ### I want better completions, should I use GPT-4? -Perhaps surprisingly, the answer is no. The models that we suggest for autocomplete are trained with a highly specific prompt format, which allows them to respond to requests for completing code (see examples of these prompts [here](https://github.com/continuedev/continue/blob/d2bc6359e8ebf647892ec953e418042dc7f8a685/core/autocomplete/templates.ts)). Some of the best commercial models like GPT-4 or Claude are not trained with this prompt format, which means that they won't generate useful completions. Luckily, a huge model is not required for great autocomplete. Most of the state-of-the-art autocomplete models are no more than 10b parameters, and increasing beyond this does not significantly improve performance. +Perhaps surprisingly, the answer is no. The models that we suggest for autocomplete are trained with a highly specific prompt format, which allows them to respond to requests for completing code (see examples of these prompts [here](https://github.com/continuedev/continue/blob/main/core/autocomplete/templates.ts)). Some of the best commercial models like GPT-4 or Claude are not trained with this prompt format, which means that they won't generate useful completions. Luckily, a huge model is not required for great autocomplete. Most of the state-of-the-art autocomplete models are no more than 10b parameters, and increasing beyond this does not significantly improve performance. ### I'm not seeing any completions @@ -128,7 +124,7 @@ Follow these steps to ensure that everything is set up correctly: 1. Make sure you have the "Enable Tab Autocomplete" setting checked (in VS Code, you can toggle by clicking the "Continue" button in the status bar). 2. Make sure you have downloaded Ollama. -3. Run `ollama run starcoder:3b` to verify that the model is downloaded. +3. Run `ollama run qwen2.5-coder:1.5b` to verify that the model is downloaded. 4. Make sure that any other completion providers are disabled (e.g. Copilot), as they may interfere. 5. Make sure that you aren't also using another Ollama model for chat. This will cause Ollama to constantly load and unload the models from memory, resulting in slow responses (or none at all) for both. 6. Check the output of the logs to find any potential errors (cmd/ctrl+shift+p -> "Toggle Developer Tools" -> "Console" tab in VS Code, ~/.continue/logs/core.log in JetBrains). diff --git a/core/autocomplete/charStream.test.ts b/core/autocomplete/charStream.test.ts deleted file mode 100644 index b5504ad63b..0000000000 --- a/core/autocomplete/charStream.test.ts +++ /dev/null @@ -1,160 +0,0 @@ -import { stopAtStopTokens } from "./streamTransforms/charStream"; - -describe("stopAtStopTokens", () => { - async function* createMockStream(chunks: string[]): AsyncGenerator { - for (const chunk of chunks) { - yield chunk; - } - } - - it("should yield characters until a stop token is encountered", async () => { - const mockStream = createMockStream(["Hello", " world", "! Stop", "here"]); - const stopTokens = ["Stop"]; - const result = stopAtStopTokens(mockStream, stopTokens); - - const output = []; - for await (const char of result) { - output.push(char); - } - - expect(output.join("")).toBe("Hello world! "); - }); - - it("should handle multiple stop tokens", async () => { - const mockStream = createMockStream([ - "This", - " is a ", - "test. END", - " of stream", - ]); - const stopTokens = ["END", "STOP", "HALT"]; - const result = stopAtStopTokens(mockStream, stopTokens); - - const output = []; - for await (const char of result) { - output.push(char); - } - - expect(output.join("")).toBe("This is a test. "); - }); - - it("should handle stop tokens split across chunks", async () => { - const mockStream = createMockStream([ - "Hello", - " wo", - "r", - "ld! ST", - "OP now", - ]); - const stopTokens = ["STOP"]; - const result = stopAtStopTokens(mockStream, stopTokens); - - const output = []; - for await (const char of result) { - output.push(char); - } - - expect(output.join("")).toBe("Hello world! "); - }); - - it("should yield all characters if no stop token is encountered", async () => { - const mockStream = createMockStream([ - "This", - " is ", - "a complete", - " stream", - ]); - const stopTokens = ["END"]; - const result = stopAtStopTokens(mockStream, stopTokens); - - const output = []; - for await (const char of result) { - output.push(char); - } - - expect(output.join("")).toBe("This is a complete stream"); - }); - - it("should handle empty chunks", async () => { - const mockStream = createMockStream(["Hello", "", " world", "", "! STOP"]); - const stopTokens = ["STOP"]; - const result = stopAtStopTokens(mockStream, stopTokens); - - const output = []; - for await (const char of result) { - output.push(char); - } - - expect(output.join("")).toBe("Hello world! "); - }); - - it("should handle stop token at the beginning of the stream", async () => { - const mockStream = createMockStream(["STOP", "Hello world"]); - const stopTokens = ["STOP"]; - const result = stopAtStopTokens(mockStream, stopTokens); - - const output = []; - for await (const char of result) { - output.push(char); - } - - expect(output.join("")).toBe(""); - }); - - it("should handle stop token at the end of the stream", async () => { - const mockStream = createMockStream(["Hello world", "STOP"]); - const stopTokens = ["STOP"]; - const result = stopAtStopTokens(mockStream, stopTokens); - - const output = []; - for await (const char of result) { - output.push(char); - } - - expect(output.join("")).toBe("Hello world"); - }); - - it("should handle multiple stop tokens of different lengths", async () => { - const mockStream = createMockStream([ - "This is a ", - "test with ", - "multiple STOP", - " tokens END", - ]); - const stopTokens = ["STOP", "END", "HALT"]; - const result = stopAtStopTokens(mockStream, stopTokens); - - const output = []; - for await (const char of result) { - output.push(char); - } - - expect(output.join("")).toBe("This is a test with multiple "); - }); - - it("should handle an empty stream", async () => { - const mockStream = createMockStream([]); - const stopTokens = ["STOP"]; - const result = stopAtStopTokens(mockStream, stopTokens); - - const output = []; - for await (const char of result) { - output.push(char); - } - - expect(output.join("")).toBe(""); - }); - - it("should handle an empty stop tokens array", async () => { - const mockStream = createMockStream(["Hello", " world!"]); - const stopTokens: string[] = []; - const result = stopAtStopTokens(mockStream, stopTokens); - - const output = []; - for await (const char of result) { - output.push(char); - } - - expect(output.join("")).toBe("Hello world!"); - }); -}); diff --git a/core/autocomplete/classification/shouldCompleteMultiline.ts b/core/autocomplete/classification/shouldCompleteMultiline.ts new file mode 100644 index 0000000000..75cfdcd706 --- /dev/null +++ b/core/autocomplete/classification/shouldCompleteMultiline.ts @@ -0,0 +1,53 @@ +import { AutocompleteLanguageInfo } from "../constants/AutocompleteLanguageInfo"; +import { HelperVars } from "../util/HelperVars"; + +function isMidlineCompletion(prefix: string, suffix: string): boolean { + return !suffix.startsWith("\n"); +} + +function shouldCompleteMultilineBasedOnLanguage( + language: AutocompleteLanguageInfo, + prefix: string, + suffix: string, +) { + return language.useMultiline?.({ prefix, suffix }) ?? true; +} + +export function shouldCompleteMultiline(helper: HelperVars) { + switch (helper.options.multilineCompletions) { + case "always": + return true; + case "never": + return false; + default: + break; + } + + // Always single-line if an intellisense option is selected + if (helper.input.selectedCompletionInfo) { + return true; + } + + // // Don't complete multi-line if you are mid-line + // if (isMidlineCompletion(helper.fullPrefix, helper.fullSuffix)) { + // return false; + // } + + // Don't complete multi-line for single-line comments + if ( + helper.lang.singleLineComment && + helper.fullPrefix + .split("\n") + .slice(-1)[0] + ?.trimStart() + .startsWith(helper.lang.singleLineComment) + ) { + return false; + } + + return shouldCompleteMultilineBasedOnLanguage( + helper.lang, + helper.prunedPrefix, + helper.prunedSuffix, + ); +} diff --git a/core/autocomplete/completionProvider.ts b/core/autocomplete/completionProvider.ts deleted file mode 100644 index d954c3f31c..0000000000 --- a/core/autocomplete/completionProvider.ts +++ /dev/null @@ -1,779 +0,0 @@ -import ignore from "ignore"; -import OpenAI from "openai"; -import path from "path"; -import { v4 as uuidv4 } from "uuid"; -import { RangeInFileWithContents } from "../commands/util.js"; -import { ConfigHandler } from "../config/ConfigHandler.js"; -import { TRIAL_FIM_MODEL } from "../config/onboarding.js"; -import { streamLines } from "../diff/util.js"; -import { - IDE, - ILLM, - ModelProvider, - Position, - Range, - TabAutocompleteOptions, -} from "../index.js"; -import { logDevData } from "../util/devdata.js"; -import { getBasename, getLastNPathParts } from "../util/index.js"; -import { - COUNT_COMPLETION_REJECTED_AFTER, - DEFAULT_AUTOCOMPLETE_OPTS, -} from "../util/parameters.js"; -import { Telemetry } from "../util/posthog.js"; -import { getRangeInString } from "../util/ranges.js"; - -import AutocompleteLruCache from "./cache.js"; -import { - constructAutocompletePrompt, - languageForFilepath, -} from "./constructPrompt.js"; -import { isOnlyPunctuationAndWhitespace } from "./filter.js"; -import { AutocompleteLanguageInfo } from "./languages.js"; -import { postprocessCompletion } from "./postprocessing.js"; -import { AutocompleteSnippet } from "./ranking.js"; -import { RecentlyEditedRange } from "./recentlyEdited.js"; -import { RootPathContextService } from "./services/RootPathContextService.js"; -import { - avoidPathLineAndEmptyComments, - noTopLevelKeywordsMidline, - skipPrefixes, - stopAtLines, - stopAtRepeatingLines, - stopAtSimilarLine, - streamWithNewLines, -} from "./streamTransforms/lineStream.js"; -import { getTemplateForModel } from "./templates.js"; -import { GeneratorReuseManager } from "./util.js"; -// @prettier-ignore -import Handlebars from "handlebars"; -import { getConfigJsonPath } from "../util/paths.js"; -import { BracketMatchingService } from "./services/BracketMatchingService.js"; -import { ImportDefinitionsService } from "./services/ImportDefinitionsService.js"; -import { - noFirstCharNewline, - onlyWhitespaceAfterEndOfLine, - stopAtStopTokens, -} from "./streamTransforms/charStream.js"; - -export interface AutocompleteInput { - completionId: string; - filepath: string; - pos: Position; - recentlyEditedFiles: RangeInFileWithContents[]; - recentlyEditedRanges: RecentlyEditedRange[]; - clipboardText: string; - // Used for notebook files - manuallyPassFileContents?: string; - // Used for VS Code git commit input box - manuallyPassPrefix?: string; - selectedCompletionInfo?: { - text: string; - range: Range; - }; - injectDetails?: string; -} - -export interface AutocompleteOutcome extends TabAutocompleteOptions { - accepted?: boolean; - time: number; - prefix: string; - suffix: string; - prompt: string; - completion: string; - modelProvider: string; - modelName: string; - completionOptions: any; - cacheHit: boolean; - filepath: string; - gitRepo?: string; - completionId: string; - uniqueId: string; -} - -const autocompleteCache = AutocompleteLruCache.get(); - -const DOUBLE_NEWLINE = "\n\n"; -const WINDOWS_DOUBLE_NEWLINE = "\r\n\r\n"; -const SRC_DIRECTORY = "/src/"; -// Starcoder2 tends to output artifacts starting with the letter "t" -const STARCODER2_T_ARTIFACTS = ["t.", "\nt", ""]; -const PYTHON_ENCODING = "#- coding: utf-8"; -const CODE_BLOCK_END = "```"; - -const multilineStops: string[] = [DOUBLE_NEWLINE, WINDOWS_DOUBLE_NEWLINE]; -const commonStops = [SRC_DIRECTORY, PYTHON_ENCODING, CODE_BLOCK_END]; - -// Errors that can be expected on occasion even during normal functioning should not be shown. -// Not worth disrupting the user to tell them that a single autocomplete request didn't go through -const ERRORS_TO_IGNORE = [ - // From Ollama - "unexpected server status", -]; - -function formatExternalSnippet( - filepath: string, - snippet: string, - language: AutocompleteLanguageInfo, -) { - const comment = language.singleLineComment; - const lines = [ - `${comment} Path: ${getBasename(filepath)}`, - ...snippet - .trim() - .split("\n") - .map((line) => `${comment} ${line}`), - comment, - ]; - return lines.join("\n"); -} - -let shownGptClaudeWarning = false; -const nonAutocompleteModels = [ - // "gpt", - // "claude", - "mistral", - "instruct", -]; - -export type GetLspDefinitionsFunction = ( - filepath: string, - contents: string, - cursorIndex: number, - ide: IDE, - lang: AutocompleteLanguageInfo, -) => Promise; - -export class CompletionProvider { - private static debounceTimeout: NodeJS.Timeout | undefined = undefined; - private static debouncing = false; - private static lastUUID: string | undefined = undefined; - - constructor( - private readonly configHandler: ConfigHandler, - private readonly ide: IDE, - private readonly getLlm: () => Promise, - private readonly _onError: (e: any) => void, - private readonly getDefinitionsFromLsp: GetLspDefinitionsFunction, - ) { - this.generatorReuseManager = new GeneratorReuseManager( - this.onError.bind(this), - ); - this.importDefinitionsService = new ImportDefinitionsService(this.ide); - this.rootPathContextService = new RootPathContextService( - this.importDefinitionsService, - this.ide, - ); - } - - private importDefinitionsService: ImportDefinitionsService; - private rootPathContextService: RootPathContextService; - private generatorReuseManager: GeneratorReuseManager; - private autocompleteCache = AutocompleteLruCache.get(); - public errorsShown: Set = new Set(); - private bracketMatchingService = new BracketMatchingService(); - // private nearbyDefinitionsService = new NearbyDefinitionsService(); - - private onError(e: any) { - console.warn("Error generating autocompletion: ", e); - if ( - ERRORS_TO_IGNORE.some((err) => - typeof e === "string" ? e.includes(err) : e?.message?.includes(err), - ) - ) { - return; - } - if (!this.errorsShown.has(e.message)) { - this.errorsShown.add(e.message); - this._onError(e); - } - } - - public cancel() { - this._abortControllers.forEach((abortController, id) => { - abortController.abort(); - }); - this._abortControllers.clear(); - } - - // Key is completionId - private _abortControllers = new Map(); - private _logRejectionTimeouts = new Map(); - private _outcomes = new Map(); - - public accept(completionId: string) { - if (this._logRejectionTimeouts.has(completionId)) { - clearTimeout(this._logRejectionTimeouts.get(completionId)); - this._logRejectionTimeouts.delete(completionId); - } - - if (this._outcomes.has(completionId)) { - const outcome = this._outcomes.get(completionId)!; - outcome.accepted = true; - logDevData("autocomplete", outcome); - void Telemetry.capture( - "autocomplete", - { - accepted: outcome.accepted, - modelName: outcome.modelName, - modelProvider: outcome.modelProvider, - time: outcome.time, - cacheHit: outcome.cacheHit, - }, - true, - ); - this._outcomes.delete(completionId); - - this.bracketMatchingService.handleAcceptedCompletion( - outcome.completion, - outcome.filepath, - ); - } - } - - public cancelRejectionTimeout(completionId: string) { - if (this._logRejectionTimeouts.has(completionId)) { - clearTimeout(this._logRejectionTimeouts.get(completionId)!); - this._logRejectionTimeouts.delete(completionId); - } - - if (this._outcomes.has(completionId)) { - this._outcomes.delete(completionId); - } - } - - public async provideInlineCompletionItems( - input: AutocompleteInput, - token: AbortSignal | undefined, - ): Promise { - try { - // Debounce - const uuid = uuidv4(); - CompletionProvider.lastUUID = uuid; - - const config = await this.configHandler.loadConfig(); - const options = { - ...DEFAULT_AUTOCOMPLETE_OPTS, - ...config.tabAutocompleteOptions, - }; - - // Check whether we're in the continue config.json file - if (input.filepath === getConfigJsonPath()) { - return undefined; - } - - // Check whether autocomplete is disabled for this file - if (options.disableInFiles) { - // Relative path needed for `ignore` - const workspaceDirs = await this.ide.getWorkspaceDirs(); - let filepath = input.filepath; - for (const workspaceDir of workspaceDirs) { - if (filepath.startsWith(workspaceDir)) { - filepath = path.relative(workspaceDir, filepath); - break; - } - } - - // Worst case we can check filetype glob patterns - if (filepath === input.filepath) { - filepath = getBasename(filepath); - } - - // @ts-ignore - const pattern = ignore.default().add(options.disableInFiles); - if (pattern.ignores(filepath)) { - return undefined; - } - } - - // Create abort signal if not given - if (!token) { - const controller = new AbortController(); - token = controller.signal; - this._abortControllers.set(input.completionId, controller); - } - - // Allow disabling autocomplete from config.json - if (options.disable) { - return undefined; - } - - // Debounce - if (CompletionProvider.debouncing) { - CompletionProvider.debounceTimeout?.refresh(); - const lastUUID = await new Promise((resolve) => - setTimeout(() => { - resolve(CompletionProvider.lastUUID); - }, options.debounceDelay), - ); - if (uuid !== lastUUID) { - return undefined; - } - } else { - CompletionProvider.debouncing = true; - CompletionProvider.debounceTimeout = setTimeout(async () => { - CompletionProvider.debouncing = false; - }, options.debounceDelay); - } - - // Get completion - const llm = await this.getLlm(); - - if (!llm) { - return undefined; - } - - // Ignore empty API keys for Mistral since we currently write - // a template provider without one during onboarding - if (llm.providerName === "mistral" && llm.apiKey === "") { - return undefined; - } - - // Set temperature (but don't overrride) - if (llm.completionOptions.temperature === undefined) { - llm.completionOptions.temperature = 0.01; - } - - // Set model-specific options - const LOCAL_PROVIDERS: ModelProvider[] = [ - "ollama", - "lmstudio", - "llama.cpp", - "llamafile", - "text-gen-webui", - ]; - if ( - !config.tabAutocompleteOptions?.maxPromptTokens && - LOCAL_PROVIDERS.includes(llm.providerName) - ) { - options.maxPromptTokens = 500; - } - - const outcome = await this.getTabCompletion(token, options, llm, input); - - if (!outcome?.completion) { - return undefined; - } - - // Filter out unwanted results - if (isOnlyPunctuationAndWhitespace(outcome.completion)) { - return undefined; - } - - // Do some stuff later so as not to block return. Latency matters - const completionToCache = outcome.completion; - setTimeout(async () => { - if (!outcome.cacheHit) { - (await this.autocompleteCache).put(outcome.prefix, completionToCache); - } - }, 100); - - return outcome; - } catch (e: any) { - this.onError(e); - } finally { - this._abortControllers.delete(input.completionId); - } - } - - _lastDisplayedCompletion: { id: string; displayedAt: number } | undefined = - undefined; - - markDisplayed(completionId: string, outcome: AutocompleteOutcome) { - const logRejectionTimeout = setTimeout(() => { - // Wait 10 seconds, then assume it wasn't accepted - outcome.accepted = false; - logDevData("autocomplete", outcome); - const { prompt, completion, ...restOfOutcome } = outcome; - void Telemetry.capture( - "autocomplete", - { - ...restOfOutcome, - }, - true, - ); - this._logRejectionTimeouts.delete(completionId); - }, COUNT_COMPLETION_REJECTED_AFTER); - this._outcomes.set(completionId, outcome); - this._logRejectionTimeouts.set(completionId, logRejectionTimeout); - - // If the previously displayed completion is still waiting for rejection, - // and this one is a continuation of that (the outcome.completion is the same modulo prefix) - // then we should cancel the rejection timeout - const previous = this._lastDisplayedCompletion; - const now = Date.now(); - if (previous && this._logRejectionTimeouts.has(previous.id)) { - const previousOutcome = this._outcomes.get(previous.id); - const c1 = previousOutcome?.completion.split("\n")[0] ?? ""; - const c2 = outcome.completion.split("\n")[0]; - if ( - previousOutcome && - (c1.endsWith(c2) || - c2.endsWith(c1) || - c1.startsWith(c2) || - c2.startsWith(c1)) - ) { - this.cancelRejectionTimeout(previous.id); - } else if (now - previous.displayedAt < 500) { - // If a completion isn't shown for more than - this.cancelRejectionTimeout(previous.id); - } - } - - this._lastDisplayedCompletion = { - id: completionId, - displayedAt: now, - }; - } - - async getTabCompletion( - token: AbortSignal, - options: TabAutocompleteOptions, - llm: ILLM, - input: AutocompleteInput, - ): Promise { - const startTime = Date.now(); - - const { - filepath, - pos, - recentlyEditedFiles, - recentlyEditedRanges, - clipboardText, - manuallyPassFileContents, - manuallyPassPrefix, - } = input; - const fileContents = - manuallyPassFileContents ?? (await this.ide.readFile(filepath)); - const fileLines = fileContents.split("\n"); - - // Filter - const lang = languageForFilepath(filepath); - const line = fileLines[pos.line] ?? ""; - for (const endOfLine of lang.endOfLine) { - if (line.endsWith(endOfLine) && pos.character >= line.length) { - return undefined; - } - } - - // Model - if (!llm) { - return; - } - if (llm instanceof OpenAI) { - llm.useLegacyCompletionsEndpoint = true; - } else if ( - llm.providerName === "free-trial" && - llm.model !== TRIAL_FIM_MODEL - ) { - llm.model = TRIAL_FIM_MODEL; - } - - if ( - !shownGptClaudeWarning && - nonAutocompleteModels.some((model) => llm.model.includes(model)) && - !llm.model.toLowerCase().includes("deepseek") && - !llm.model.toLowerCase().includes("codestral") - ) { - shownGptClaudeWarning = true; - throw new Error( - `Warning: ${llm.model} is not trained for tab-autocomplete, and will result in low-quality suggestions. See the docs to learn more about why: https://docs.continue.dev/features/tab-autocomplete#i-want-better-completions-should-i-use-gpt-4`, - ); - } - - // Prompt - let fullPrefix = - getRangeInString(fileContents, { - start: { line: 0, character: 0 }, - end: input.selectedCompletionInfo?.range.start ?? pos, - }) + (input.selectedCompletionInfo?.text ?? ""); - - if (input.injectDetails) { - const lines = fullPrefix.split("\n"); - fullPrefix = `${lines.slice(0, -1).join("\n")}\n${ - lang.singleLineComment - } ${input.injectDetails - .split("\n") - .join(`\n${lang.singleLineComment} `)}\n${lines[lines.length - 1]}`; - } - - const fullSuffix = getRangeInString(fileContents, { - start: pos, - end: { line: fileLines.length - 1, character: Number.MAX_SAFE_INTEGER }, - }); - - // First non-whitespace line below the cursor - let lineBelowCursor = ""; - let i = 1; - while ( - lineBelowCursor.trim() === "" && - pos.line + i <= fileLines.length - 1 - ) { - lineBelowCursor = fileLines[Math.min(pos.line + i, fileLines.length - 1)]; - i++; - } - - let extrasSnippets = options.useOtherFiles - ? ((await Promise.race([ - this.getDefinitionsFromLsp( - filepath, - fullPrefix + fullSuffix, - fullPrefix.length, - this.ide, - lang, - ), - new Promise((resolve) => { - setTimeout(() => resolve([]), 100); - }), - ])) as AutocompleteSnippet[]) - : []; - - const workspaceDirs = await this.ide.getWorkspaceDirs(); - if (options.onlyMyCode) { - extrasSnippets = extrasSnippets.filter((snippet) => { - return workspaceDirs.some((dir) => snippet.filepath.startsWith(dir)); - }); - } - - let { prefix, suffix, completeMultiline, snippets } = - await constructAutocompletePrompt( - filepath, - pos.line, - fullPrefix, - fullSuffix, - clipboardText, - lang, - options, - recentlyEditedRanges, - recentlyEditedFiles, - llm.model, - extrasSnippets, - this.importDefinitionsService, - this.rootPathContextService, - ); - - // If prefix is manually passed - if (manuallyPassPrefix) { - prefix = manuallyPassPrefix; - suffix = ""; - } - - // Template prompt - const { - template, - completionOptions, - compilePrefixSuffix = undefined, - } = options.template - ? { template: options.template, completionOptions: {} } - : getTemplateForModel(llm.model); - - let prompt: string; - const filename = getBasename(filepath); - const reponame = getBasename(workspaceDirs[0] ?? "myproject"); - - // Some models have prompts that need two passes. This lets us pass the compiled prefix/suffix - // into either the 2nd template to generate a raw string, or to pass prefix, suffix to a FIM endpoint - if (compilePrefixSuffix) { - [prefix, suffix] = compilePrefixSuffix( - prefix, - suffix, - filepath, - reponame, - snippets, - ); - } - - if (typeof template === "string") { - const compiledTemplate = Handlebars.compile(template); - - // Format snippets as comments and prepend to prefix - const formattedSnippets = snippets - .map((snippet) => - formatExternalSnippet(snippet.filepath, snippet.contents, lang), - ) - .join("\n"); - if (formattedSnippets.length > 0) { - prefix = `${formattedSnippets}\n\n${prefix}`; - } else if (prefix.trim().length === 0 && suffix.trim().length === 0) { - // If it's an empty file, include the file name as a comment - prefix = `${lang.singleLineComment} ${getLastNPathParts( - filepath, - 2, - )}\n${prefix}`; - } - - prompt = compiledTemplate({ - prefix, - suffix, - filename, - reponame, - language: lang.name, - }); - } else { - // Let the template function format snippets - prompt = template( - prefix, - suffix, - filepath, - reponame, - lang.name, - snippets, - ); - } - - // Completion - let completion = ""; - - const cache = await autocompleteCache; - const cachedCompletion = options.useCache - ? await cache.get(prefix) - : undefined; - let cacheHit = false; - if (cachedCompletion) { - // Cache - cacheHit = true; - completion = cachedCompletion; - } else { - const stop = [ - ...(completionOptions?.stop || []), - ...multilineStops, - ...commonStops, - ...(llm.model.toLowerCase().includes("starcoder2") - ? STARCODER2_T_ARTIFACTS - : []), - ...(lang.stopWords ?? []), - ...lang.topLevelKeywords.map((word) => `\n${word}`), - ]; - - let langMultilineDecision = lang.useMultiline?.({ prefix, suffix }); - let multiline: boolean = false; - if (langMultilineDecision) { - multiline = langMultilineDecision; - } else { - multiline = - !input.selectedCompletionInfo && // Only ever single-line if using intellisense selected value - options.multilineCompletions !== "never" && - (options.multilineCompletions === "always" || completeMultiline); - } - - // Try to reuse pending requests if what the user typed matches start of completion - const generator = this.generatorReuseManager.getGenerator( - prefix, - () => - llm.supportsFim() - ? llm.streamFim(prefix, suffix, { - ...completionOptions, - stop, - }) - : llm.streamComplete(prompt, { - ...completionOptions, - raw: true, - stop, - }), - multiline, - ); - - // Full stop means to stop the LLM's generation, instead of just truncating the displayed completion - const fullStop = () => - this.generatorReuseManager.currentGenerator?.cancel(); - - // LLM - let cancelled = false; - const generatorWithCancellation = async function* () { - for await (const update of generator) { - if (token.aborted) { - cancelled = true; - return; - } - yield update; - } - }; - let charGenerator = generatorWithCancellation(); - charGenerator = noFirstCharNewline(charGenerator); - charGenerator = onlyWhitespaceAfterEndOfLine( - charGenerator, - lang.endOfLine, - fullStop, - ); - charGenerator = stopAtStopTokens(charGenerator, stop); - charGenerator = this.bracketMatchingService.stopOnUnmatchedClosingBracket( - charGenerator, - prefix, - suffix, - filepath, - multiline, - ); - - let lineGenerator = streamLines(charGenerator); - lineGenerator = stopAtLines(lineGenerator, fullStop); - lineGenerator = stopAtRepeatingLines(lineGenerator, fullStop); - lineGenerator = avoidPathLineAndEmptyComments( - lineGenerator, - lang.singleLineComment, - ); - lineGenerator = skipPrefixes(lineGenerator); - lineGenerator = noTopLevelKeywordsMidline( - lineGenerator, - lang.topLevelKeywords, - fullStop, - ); - - for (const lineFilter of lang.lineFilters ?? []) { - lineGenerator = lineFilter({ lines: lineGenerator, fullStop }); - } - - lineGenerator = streamWithNewLines(lineGenerator); - - const finalGenerator = stopAtSimilarLine( - lineGenerator, - lineBelowCursor, - fullStop, - ); - - try { - for await (const update of finalGenerator) { - completion += update; - } - } catch (e: any) { - if (ERRORS_TO_IGNORE.some((err) => e.includes(err))) { - return undefined; - } - throw e; - } - - if (cancelled) { - return undefined; - } - - const processedCompletion = postprocessCompletion({ - completion, - prefix, - suffix, - llm, - }); - - if (!processedCompletion) { - return undefined; - } - completion = processedCompletion; - } - - const time = Date.now() - startTime; - return { - time, - completion, - prefix, - suffix, - prompt, - modelProvider: llm.providerName, - modelName: llm.model, - completionOptions, - cacheHit, - filepath: input.filepath, - completionId: input.completionId, - gitRepo: await this.ide.getRepoName(input.filepath), - uniqueId: await this.ide.getUniqueId(), - ...options, - }; - } -} diff --git a/core/autocomplete/languages.ts b/core/autocomplete/constants/AutocompleteLanguageInfo.ts similarity index 85% rename from core/autocomplete/languages.ts rename to core/autocomplete/constants/AutocompleteLanguageInfo.ts index be414098a1..2f21c5a3e9 100644 --- a/core/autocomplete/languages.ts +++ b/core/autocomplete/constants/AutocompleteLanguageInfo.ts @@ -1,16 +1,17 @@ -import { LineFilter } from "./streamTransforms/lineStream"; +import { BracketMatchingService } from "../filtering/BracketMatchingService"; +import { + CharacterFilter, + LineFilter, +} from "../filtering/streamTransforms/lineStream"; export interface AutocompleteLanguageInfo { name: string; topLevelKeywords: string[]; - singleLineComment: string; + singleLineComment?: string; endOfLine: string[]; - stopWords?: string[]; lineFilters?: LineFilter[]; - useMultiline?: (args: { - prefix: string; - suffix: string; - }) => boolean | undefined; + charFilters?: CharacterFilter[]; + useMultiline?: (args: { prefix: string; suffix: string }) => boolean; } // TypeScript @@ -26,7 +27,7 @@ export const Python = { name: "Python", // """"#" is for .ipynb files, where we add '"""' surrounding markdown blocks. // This stops the model from trying to complete the start of a new markdown block - topLevelKeywords: ["def", "class", '"""#'], + topLevelKeywords: ["def", "class", "\"\"\"#"], singleLineComment: "#", endOfLine: [], }; @@ -256,10 +257,8 @@ export const YAML: AutocompleteLanguageInfo = { } else { seenListItem = true; } - yield line; - } else { - yield line; } + yield line; } }, // Don't allow consecutive lines of same key @@ -268,18 +267,39 @@ export const YAML: AutocompleteLanguageInfo = { for await (const line of lines) { if (line.includes(":")) { const key = line.split(":")[0]; - if (key !== lastKey) { + if (key === lastKey) { + break; + } else { yield line; lastKey = key; - } else { - break; } + } else { + yield line; } } }, ], }; +export const Json: AutocompleteLanguageInfo = { + name: "JSON", + topLevelKeywords: [], + singleLineComment: "//", + endOfLine: [",", "}", "]"], + charFilters: [ + function matchBrackets({ chars, prefix, suffix, filepath, multiline }) { + const bracketMatchingService = new BracketMatchingService(); + return bracketMatchingService.stopOnUnmatchedClosingBracket( + chars, + prefix, + suffix, + filepath, + multiline, + ); + }, + ], +}; + export const Markdown: AutocompleteLanguageInfo = { name: "Markdown", topLevelKeywords: [], @@ -289,7 +309,7 @@ export const Markdown: AutocompleteLanguageInfo = { const singleLineStarters = ["- ", "* ", /^\d+\. /, "> ", "```", /^#{1,6} /]; let currentLine = prefix.split("\n").pop(); if (!currentLine) { - return undefined; + return true; } currentLine = currentLine.trim(); for (const starter of singleLineStarters) { @@ -301,7 +321,7 @@ export const Markdown: AutocompleteLanguageInfo = { return false; } } - return undefined; + return true; }, }; @@ -309,6 +329,7 @@ export const LANGUAGES: { [extension: string]: AutocompleteLanguageInfo } = { ts: Typescript, js: Typescript, tsx: Typescript, + json: Json, jsx: Typescript, ipynb: Python, py: Python, @@ -346,3 +367,9 @@ export const LANGUAGES: { [extension: string]: AutocompleteLanguageInfo } = { yml: YAML, md: Markdown, }; + +export function languageForFilepath( + filepath: string, +): AutocompleteLanguageInfo { + return LANGUAGES[filepath.split(".").slice(-1)[0]] || Typescript; +} diff --git a/core/autocomplete/constructPrompt.ts b/core/autocomplete/constructPrompt.ts deleted file mode 100644 index f3d08ff5cc..0000000000 --- a/core/autocomplete/constructPrompt.ts +++ /dev/null @@ -1,287 +0,0 @@ -import { RangeInFileWithContents } from "../commands/util.js"; -import { TabAutocompleteOptions } from "../index.js"; - -import { - countTokens, - pruneLinesFromBottom, - pruneLinesFromTop, -} from "../llm/countTokens.js"; -import { AstPath, getAst, getTreePathAtCursor } from "./ast.js"; -import { - AutocompleteLanguageInfo, - LANGUAGES, - Typescript, -} from "./languages.js"; -import { - fillPromptWithSnippets, - getSymbolsForSnippet, - rankSnippets, - removeRangeFromSnippets, - type AutocompleteSnippet, -} from "./ranking.js"; -import { RecentlyEditedRange, findMatchingRange } from "./recentlyEdited.js"; -import { ImportDefinitionsService } from "./services/ImportDefinitionsService.js"; -import { RootPathContextService } from "./services/RootPathContextService.js"; - -export function languageForFilepath( - filepath: string, -): AutocompleteLanguageInfo { - return LANGUAGES[filepath.split(".").slice(-1)[0]] || Typescript; -} - -const BLOCK_TYPES = ["body", "statement_block"]; - -function shouldCompleteMultilineAst( - treePath: AstPath, - cursorLine: number, -): boolean { - // If at the base of the file, do multiline - if (treePath.length === 1) { - return true; - } - - // If at the first line of an otherwise empty funtion body, do multiline - for (let i = treePath.length - 1; i >= 0; i--) { - const node = treePath[i]; - if ( - BLOCK_TYPES.includes(node.type) && - Math.abs(node.startPosition.row - cursorLine) <= 1 - ) { - let text = node.text; - text = text.slice(text.indexOf("{") + 1); - text = text.slice(0, text.lastIndexOf("}")); - text = text.trim(); - return text.split("\n").length === 1; - } - } - - return false; -} - -function isMidlineCompletion(prefix: string, suffix: string): boolean { - return !suffix.startsWith("\n"); -} - -async function shouldCompleteMultiline( - treePath: AstPath | undefined, - fullPrefix: string, - fullSuffix: string, - language: AutocompleteLanguageInfo, -): Promise { - // Don't complete multi-line if you are mid-line - if (isMidlineCompletion(fullPrefix, fullSuffix)) { - return false; - } - - // Don't complete multi-line for single-line comments - if ( - fullPrefix - .split("\n") - .slice(-1)[0] - ?.trimStart() - .startsWith(language.singleLineComment) - ) { - return false; - } - - // First, if the line before ends with an opening bracket, then assume multi-line - if ( - ["{", "(", "["].includes( - fullPrefix.split("\n").slice(-2)[0]?.trim().slice(-1)[0], - ) - ) { - return true; - } - - // Use AST to determine whether to complete multiline - let completeMultiline = false; - if (treePath) { - const cursorLine = fullPrefix.split("\n").length - 1; - completeMultiline = shouldCompleteMultilineAst(treePath, cursorLine); - } - return completeMultiline; -} - -export async function constructAutocompletePrompt( - filepath: string, - cursorLine: number, - fullPrefix: string, - fullSuffix: string, - clipboardText: string, - language: AutocompleteLanguageInfo, - options: TabAutocompleteOptions, - recentlyEditedRanges: RecentlyEditedRange[], - recentlyEditedFiles: RangeInFileWithContents[], - modelName: string, - extraSnippets: AutocompleteSnippet[], - importDefinitionsService: ImportDefinitionsService, - rootPathContextService: RootPathContextService, -): Promise<{ - prefix: string; - suffix: string; - useFim: boolean; - completeMultiline: boolean; - snippets: AutocompleteSnippet[]; -}> { - // Construct basic prefix - const maxPrefixTokens = options.maxPromptTokens * options.prefixPercentage; - const prefix = pruneLinesFromTop(fullPrefix, maxPrefixTokens, modelName); - - // Construct suffix - const maxSuffixTokens = Math.min( - options.maxPromptTokens - countTokens(prefix, modelName), - options.maxSuffixPercentage * options.maxPromptTokens, - ); - const suffix = pruneLinesFromBottom(fullSuffix, maxSuffixTokens, modelName); - - // Calculate AST Path - let treePath: AstPath | undefined; - try { - const ast = await getAst(filepath, fullPrefix + fullSuffix); - if (ast) { - treePath = await getTreePathAtCursor(ast, fullPrefix.length); - } - } catch (e) { - console.error("Failed to parse AST", e); - } - - // Find external snippets - let snippets: AutocompleteSnippet[] = []; - - if (options.useOtherFiles) { - snippets.push(...extraSnippets); - - const windowAroundCursor = - fullPrefix.slice( - -options.slidingWindowSize * options.slidingWindowPrefixPercentage, - ) + - fullSuffix.slice( - options.slidingWindowSize * (1 - options.slidingWindowPrefixPercentage), - ); - - // This was much too slow, and not super useful - // const slidingWindowMatches = await slidingWindowMatcher( - // recentlyEditedFiles, - // windowAroundCursor, - // 3, - // options.slidingWindowSize, - // ); - // snippets.push(...slidingWindowMatches); - - // snippets.push( - // ...recentlyEditedRanges.map((r) => ({ - // ...r, - // contents: r.lines.join("\n"), - // })), - // ); - - if (options.useRecentlyEdited) { - const currentLinePrefix = prefix.trim().split("\n").slice(-1)[0]; - if (currentLinePrefix?.length > options.recentLinePrefixMatchMinLength) { - const matchingRange = findMatchingRange( - recentlyEditedRanges, - currentLinePrefix, - ); - if (matchingRange) { - snippets.push({ - ...matchingRange, - contents: matchingRange.lines.join("\n"), - score: 0.8, - }); - } - } - } - - // Use imports - if (options.useImports) { - const importSnippets = []; - const fileInfo = importDefinitionsService.get(filepath); - if (fileInfo) { - const { imports } = fileInfo; - // Look for imports of any symbols around the current range - const textAroundCursor = - fullPrefix.split("\n").slice(-5).join("\n") + - fullSuffix.split("\n").slice(0, 3).join("\n"); - const symbols = Array.from( - getSymbolsForSnippet(textAroundCursor), - ).filter((symbol) => !language.topLevelKeywords.includes(symbol)); - for (const symbol of symbols) { - const rifs = imports[symbol]; - if (Array.isArray(rifs)) { - importSnippets.push(...rifs); - } - } - } - snippets.push(...importSnippets); - } - - if (options.useRootPathContext && treePath) { - const ctx = await rootPathContextService.getContextForPath( - filepath, - treePath, - ); - snippets.push(...ctx); - } - - // Filter out empty snippets and ones that are already in the prefix/suffix - snippets = snippets - .map((snippet) => ({ ...snippet })) - .filter( - (s) => - s.contents.trim() !== "" && - !(prefix + suffix).includes(s.contents.trim()), - ); - - // Rank / order the snippets - const scoredSnippets = rankSnippets(snippets, windowAroundCursor); - - // Fill maxSnippetTokens with snippets - const maxSnippetTokens = - options.maxPromptTokens * options.maxSnippetPercentage; - - // Remove prefix range from snippets - const prefixLines = prefix.split("\n").length; - const suffixLines = suffix.split("\n").length; - const buffer = 8; - const prefixSuffixRangeWithBuffer = { - start: { - line: cursorLine - prefixLines - buffer, - character: 0, - }, - end: { - line: cursorLine + suffixLines + buffer, - character: 0, - }, - }; - let finalSnippets = removeRangeFromSnippets( - scoredSnippets, - filepath.split("://").slice(-1)[0], - prefixSuffixRangeWithBuffer, - ); - - // Filter snippets for those with best scores (must be above threshold) - finalSnippets = finalSnippets.filter( - (snippet) => snippet.score >= options.recentlyEditedSimilarityThreshold, - ); - finalSnippets = fillPromptWithSnippets( - scoredSnippets, - maxSnippetTokens, - modelName, - ); - - snippets = finalSnippets; - } - - return { - prefix, - suffix, - useFim: true, - completeMultiline: await shouldCompleteMultiline( - treePath, - fullPrefix, - fullSuffix, - language, - ), - snippets, - }; -} diff --git a/core/autocomplete/context/ContextRetrievalService.ts b/core/autocomplete/context/ContextRetrievalService.ts new file mode 100644 index 0000000000..72e061fed1 --- /dev/null +++ b/core/autocomplete/context/ContextRetrievalService.ts @@ -0,0 +1,73 @@ +import { IDE } from "../.."; +import { + AutocompleteCodeSnippet, + AutocompleteSnippetType, +} from "../snippets/types"; +import { HelperVars } from "../util/HelperVars"; + +import { ImportDefinitionsService } from "./ImportDefinitionsService"; +import { getSymbolsForSnippet } from "./ranking"; +import { RootPathContextService } from "./root-path-context/RootPathContextService"; + +export class ContextRetrievalService { + private importDefinitionsService: ImportDefinitionsService; + private rootPathContextService: RootPathContextService; + + constructor(private readonly ide: IDE) { + this.importDefinitionsService = new ImportDefinitionsService(this.ide); + this.rootPathContextService = new RootPathContextService( + this.importDefinitionsService, + this.ide, + ); + } + + public async getSnippetsFromImportDefinitions( + helper: HelperVars, + ): Promise { + if (helper.options.useImports === false) { + return []; + } + + const importSnippets: AutocompleteCodeSnippet[] = []; + const fileInfo = this.importDefinitionsService.get(helper.filepath); + if (fileInfo) { + const { imports } = fileInfo; + // Look for imports of any symbols around the current range + const textAroundCursor = + helper.fullPrefix.split("\n").slice(-5).join("\n") + + helper.fullSuffix.split("\n").slice(0, 3).join("\n"); + const symbols = Array.from(getSymbolsForSnippet(textAroundCursor)).filter( + (symbol) => !helper.lang.topLevelKeywords.includes(symbol), + ); + for (const symbol of symbols) { + const rifs = imports[symbol]; + if (Array.isArray(rifs)) { + const snippets: AutocompleteCodeSnippet[] = rifs.map((rif) => { + return { + filepath: rif.filepath, + content: rif.contents, + type: AutocompleteSnippetType.Code, + }; + }); + + importSnippets.push(...snippets); + } + } + } + + return importSnippets; + } + + public async getRootPathSnippets( + helper: HelperVars, + ): Promise { + if (!helper.treePath) { + return []; + } + + return this.rootPathContextService.getContextForPath( + helper.filepath, + helper.treePath, + ); + } +} diff --git a/core/autocomplete/services/ImportDefinitionsService.ts b/core/autocomplete/context/ImportDefinitionsService.ts similarity index 78% rename from core/autocomplete/services/ImportDefinitionsService.ts rename to core/autocomplete/context/ImportDefinitionsService.ts index 0df08f0968..309310738b 100644 --- a/core/autocomplete/services/ImportDefinitionsService.ts +++ b/core/autocomplete/context/ImportDefinitionsService.ts @@ -1,8 +1,7 @@ -import { IDE } from "../.."; -import { RangeInFileWithContents } from "../../commands/util"; +import { IDE, RangeInFileWithContents } from "../.."; import { PrecalculatedLruCache } from "../../util/LruCache"; import { - TSQueryType, + getFullLanguageName, getParserForFile, getQueryForFile, } from "../../util/treeSitter"; @@ -30,14 +29,23 @@ export class ImportDefinitionsService { return this.cache.get(filepath); } - private async _getFileInfo(filepath: string): Promise { + private async _getFileInfo(filepath: string): Promise { const parser = await getParserForFile(filepath); if (!parser) { return { imports: {}, }; } - const ast = parser.parse(await this.ide.readFile(filepath), undefined, { + + let fileContents: string | undefined = undefined; + try { + fileContents = await this.ide.readFile(filepath); + } catch (err) { + // File removed + return null; + } + + const ast = parser.parse(fileContents, undefined, { includedRanges: [ { startIndex: 0, @@ -47,7 +55,11 @@ export class ImportDefinitionsService { }, ], }); - const query = await getQueryForFile(filepath, TSQueryType.Imports); + const language = getFullLanguageName(filepath); + const query = await getQueryForFile( + filepath, + `import-queries/${language}.scm`, + ); if (!query) { return { imports: {}, diff --git a/core/autocomplete/context/ranking/index.ts b/core/autocomplete/context/ranking/index.ts new file mode 100644 index 0000000000..1064fd2436 --- /dev/null +++ b/core/autocomplete/context/ranking/index.ts @@ -0,0 +1,155 @@ +import { RangeInFileWithContents } from "../../../"; +import { countTokens } from "../../../llm/countTokens"; +import { AutocompleteSnippetDeprecated } from "../../types"; +import { HelperVars } from "../../util/HelperVars"; + +const rx = /[\s.,\/#!$%\^&\*;:{}=\-_`~()\[\]]/g; +export function getSymbolsForSnippet(snippet: string): Set { + const symbols = snippet + .split(rx) + .map((s) => s.trim()) + .filter((s) => s !== ""); + return new Set(symbols); +} + +/** + * Calculate similarity as number of shared symbols divided by total number of unique symbols between both. + */ +function jaccardSimilarity(a: string, b: string): number { + const aSet = getSymbolsForSnippet(a); + const bSet = getSymbolsForSnippet(b); + const union = new Set([...aSet, ...bSet]).size; + + // Avoid division by zero + if (union === 0) { + return 0; + } + + let intersection = 0; + for (const symbol of aSet) { + if (bSet.has(symbol)) { + intersection++; + } + } + + return intersection / union; +} + +/** + * Rank code snippets to be used in tab-autocomplete prompt. Returns a sorted version of the snippet array. + */ +export function rankAndOrderSnippets( + ranges: AutocompleteSnippetDeprecated[], + helper: HelperVars, +): Required[] { + const windowAroundCursor = + helper.fullPrefix.slice( + -helper.options.slidingWindowSize * + helper.options.slidingWindowPrefixPercentage, + ) + + helper.fullSuffix.slice( + helper.options.slidingWindowSize * + (1 - helper.options.slidingWindowPrefixPercentage), + ); + + const snippets: Required[] = ranges.map( + (snippet) => ({ + score: + snippet.score ?? + jaccardSimilarity(snippet.contents, windowAroundCursor), + ...snippet, + }), + ); + const uniqueSnippets = deduplicateSnippets(snippets); + return uniqueSnippets.sort((a, b) => a.score - b.score); +} + +/** + * Deduplicate code snippets by merging overlapping ranges into a single range. + */ +function deduplicateSnippets( + snippets: Required[], +): Required[] { + // Group by file + const fileGroups: { + [key: string]: Required[]; + } = {}; + for (const snippet of snippets) { + if (!fileGroups[snippet.filepath]) { + fileGroups[snippet.filepath] = []; + } + fileGroups[snippet.filepath].push(snippet); + } + + // Merge overlapping ranges + const allRanges = []; + for (const file of Object.keys(fileGroups)) { + allRanges.push(...mergeSnippetsByRange(fileGroups[file])); + } + return allRanges; +} + +function mergeSnippetsByRange( + snippets: Required[], +): Required[] { + if (snippets.length <= 1) { + return snippets; + } + + const sorted = snippets.sort( + (a, b) => a.range.start.line - b.range.start.line, + ); + const merged: Required[] = []; + + while (sorted.length > 0) { + const next = sorted.shift()!; + const last = merged[merged.length - 1]; + if (merged.length > 0 && last.range.end.line >= next.range.start.line) { + // Merge with previous snippet + last.score = Math.max(last.score, next.score); + try { + last.range.end = next.range.end; + } catch (e) { + console.log("Error merging ranges", e); + } + last.contents = mergeOverlappingRangeContents(last, next); + } else { + merged.push(next); + } + } + + return merged; +} + +function mergeOverlappingRangeContents( + first: RangeInFileWithContents, + second: RangeInFileWithContents, +): string { + const firstLines = first.contents.split("\n"); + const numOverlapping = first.range.end.line - second.range.start.line; + return `${firstLines.slice(-numOverlapping).join("\n")}\n${second.contents}`; +} + +/** + * Fill the allowed space with snippets. + * It is assumed that the snippets are sorted by score. + */ +export function fillPromptWithSnippets( + snippets: Required[], + maxSnippetTokens: number, + modelName: string, +): Required[] { + let tokensRemaining = maxSnippetTokens; + const keptSnippets: Required[] = []; + for (let i = 0; i < snippets.length; i++) { + const snippet = snippets[i]; + const tokenCount = countTokens(snippet.contents, modelName); + if (tokensRemaining - tokenCount >= 0) { + tokensRemaining -= tokenCount; + keptSnippets.push(snippet); + } else { + } + } + + return keptSnippets; +} diff --git a/core/autocomplete/context/root-path-context/RootPathContextService.ts b/core/autocomplete/context/root-path-context/RootPathContextService.ts new file mode 100644 index 0000000000..93648653fd --- /dev/null +++ b/core/autocomplete/context/root-path-context/RootPathContextService.ts @@ -0,0 +1,193 @@ +import { createHash } from "crypto"; + +import { LRUCache } from "lru-cache"; +import Parser from "web-tree-sitter"; + +import { IDE } from "../../.."; +import { + getFullLanguageName, + getQueryForFile, + IGNORE_PATH_PATTERNS, + LanguageName, +} from "../../../util/treeSitter"; +import { + AutocompleteCodeSnippet, + AutocompleteSnippetType, +} from "../../snippets/types"; +import { AutocompleteSnippetDeprecated } from "../../types"; +import { AstPath } from "../../util/ast"; +import { ImportDefinitionsService } from "../ImportDefinitionsService"; + +function getSyntaxTreeString( + node: Parser.SyntaxNode, + indent: string = "", +): string { + let result = ""; + const nodeInfo = `${node.type} [${node.startPosition.row}:${node.startPosition.column} - ${node.endPosition.row}:${node.endPosition.column}]`; + result += `${indent}${nodeInfo}\n`; + + for (const child of node.children) { + result += getSyntaxTreeString(child, indent + " "); + } + + return result; +} + +export class RootPathContextService { + private cache = new LRUCache({ + max: 100, + }); + + constructor( + private readonly importDefinitionsService: ImportDefinitionsService, + private readonly ide: IDE, + ) {} + + private static getNodeId(node: Parser.SyntaxNode): string { + return `${node.startIndex}`; + } + + private static TYPES_TO_USE = new Set([ + "arrow_function", + "generator_function_declaration", + "program", + "function_declaration", + "function_definition", + "method_definition", + "method_declaration", + "class_declaration", + "class_definition", + ]); + + /** + * Key comes from hash of parent key and node type and node id. + */ + private static keyFromNode( + parentKey: string, + astNode: Parser.SyntaxNode, + ): string { + return createHash("sha256") + .update(parentKey) + .update(astNode.type) + .update(RootPathContextService.getNodeId(astNode)) + .digest("hex"); + } + + private async getSnippetsForNode( + filepath: string, + node: Parser.SyntaxNode, + ): Promise { + const snippets: AutocompleteSnippetDeprecated[] = []; + const language = getFullLanguageName(filepath); + + let query: Parser.Query | undefined; + switch (node.type) { + case "program": + this.importDefinitionsService.get(filepath); + break; + default: + // const type = node.type; + // console.log(getSyntaxTreeString(node)); + // debugger; + + query = await getQueryForFile( + filepath, + `root-path-context-queries/${language}/${node.type}.scm`, + ); + break; + } + + if (!query) { + return snippets; + } + + const queries = query.matches(node).map(async (match) => { + for (const item of match.captures) { + try { + const endPosition = item.node.endPosition; + const newSnippets = await this.getSnippets( + filepath, + endPosition, + language, + ); + snippets.push(...newSnippets); + } catch (e) { + throw e; + } + } + }); + + await Promise.all(queries); + + return snippets; + } + + private async getSnippets( + filepath: string, + endPosition: Parser.Point, + language: LanguageName, + ): Promise { + const definitions = await this.ide.gotoDefinition({ + filepath, + position: { + line: endPosition.row, + character: endPosition.column, + }, + }); + const newSnippets = await Promise.all( + definitions + .filter((definition) => { + const isIgnoredPath = IGNORE_PATH_PATTERNS[language]?.some( + (pattern) => pattern.test(definition.filepath), + ); + + return !isIgnoredPath; + }) + .map(async (def) => ({ + ...def, + contents: await this.ide.readRangeInFile(def.filepath, def.range), + })), + ); + + return newSnippets; + } + + async getContextForPath( + filepath: string, + astPath: AstPath, + // cursorIndex: number, + ): Promise { + const snippets: AutocompleteCodeSnippet[] = []; + + let parentKey = filepath; + for (const astNode of astPath.filter((node) => + RootPathContextService.TYPES_TO_USE.has(node.type), + )) { + const key = RootPathContextService.keyFromNode(parentKey, astNode); + // const type = astNode.type; + // debugger; + + const foundInCache = this.cache.get(key); + const newSnippets = + foundInCache ?? (await this.getSnippetsForNode(filepath, astNode)); + + const formattedSnippets: AutocompleteCodeSnippet[] = newSnippets.map( + (item) => ({ + filepath: item.filepath, + content: item.contents, + type: AutocompleteSnippetType.Code, + }), + ); + + snippets.push(...formattedSnippets); + + if (!foundInCache) { + this.cache.set(key, newSnippets); + } + + parentKey = key; + } + + return snippets; + } +} diff --git a/core/autocomplete/context/root-path-context/test/RootPathContextService.test.ts b/core/autocomplete/context/root-path-context/test/RootPathContextService.test.ts new file mode 100644 index 0000000000..a954dda845 --- /dev/null +++ b/core/autocomplete/context/root-path-context/test/RootPathContextService.test.ts @@ -0,0 +1,56 @@ +import { PYTHON_TEST_CASES, TYPESCRIPT_TEST_CASES } from "./testCases"; +import { testRootPathContext } from "./testUtils"; + +const TEST_CASES = [ + ...PYTHON_TEST_CASES, + ...TYPESCRIPT_TEST_CASES, + { + nodeType: "function_definition", + fileName: "file1.php", + language: "PHP", + cursorPosition: { line: 12, character: 32 }, + definitionPositions: [ + { row: 10, column: 26 }, // Person + { row: 10, column: 44 }, // Address + ], + }, + { + nodeType: "method_declaration", + fileName: "file1.php", + language: "PHP", + cursorPosition: { line: 26, character: 35 }, + definitionPositions: [ + { row: 15, column: 29 }, // BaseClass + { row: 15, column: 55 }, // FirstInterface + { row: 15, column: 72 }, // SecondInterface + { row: 25, column: 43 }, // Person + { row: 25, column: 61 }, // Address + ], + }, + { + nodeType: "function_declaration", + fileName: "file1.go", + language: "Go", + cursorPosition: { line: 7, character: 21 }, + definitionPositions: [ + { row: 6, column: 33 }, // models.User + { row: 6, column: 50 }, // models.Address + ], + }, +]; + +describe("RootPathContextService", () => { + describe("should look for correct type definitions", () => { + test.each(TEST_CASES)( + "$language: $nodeType", + async ({ fileName, cursorPosition, definitionPositions }) => { + await testRootPathContext( + "files", + fileName, + cursorPosition, + definitionPositions, + ); + }, + ); + }); +}); diff --git a/core/autocomplete/context/root-path-context/test/files/__init__.py b/core/autocomplete/context/root-path-context/test/files/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/core/autocomplete/context/root-path-context/test/files/base_module.py b/core/autocomplete/context/root-path-context/test/files/base_module.py new file mode 100644 index 0000000000..cf0dc0abb6 --- /dev/null +++ b/core/autocomplete/context/root-path-context/test/files/base_module.py @@ -0,0 +1,26 @@ +# File: base_module.py + +class BaseClass: + def __init__(self): + print("BaseClass initialized") + +class Collection: + def __init__(self): + print("Collection initialized") + +class Address: + def __init__(self, street: str, city: str, zip_code: str): + self.street = street + self.city = city + self.zip_code = zip_code + + def __str__(self): + return f"{self.street}, {self.city}, {self.zip_code}" + +class Person: + def __init__(self, name: str, address: Address): + self.name = name + self.address = address + + def __str__(self): + return f"{self.name} lives at {self.address}" diff --git a/core/autocomplete/context/root-path-context/test/files/file1.go b/core/autocomplete/context/root-path-context/test/files/file1.go new file mode 100644 index 0000000000..94eec171c7 --- /dev/null +++ b/core/autocomplete/context/root-path-context/test/files/file1.go @@ -0,0 +1,9 @@ +package main + +import ( + "core/autocomplete/context/root-path-context/test/files/models" +) + +func getAddress(user *models.User) *models.Address { + return user.Address +} \ No newline at end of file diff --git a/core/autocomplete/context/root-path-context/test/files/file1.php b/core/autocomplete/context/root-path-context/test/files/file1.php new file mode 100644 index 0000000000..a72f805e09 --- /dev/null +++ b/core/autocomplete/context/root-path-context/test/files/file1.php @@ -0,0 +1,32 @@ +getAddress(); +} + +class Group extends BaseClass implements FirstInterface, SecondInterface +{ + private array $people; + + public function __construct(array $people) + { + parent::__construct(); + $this->people = $people; + } + + public function getPersonAddress(Person $person): Address + { + return getAddress($person); + } +} + +?> \ No newline at end of file diff --git a/core/autocomplete/context/root-path-context/test/files/python/classes.py b/core/autocomplete/context/root-path-context/test/files/python/classes.py new file mode 100644 index 0000000000..d752d84337 --- /dev/null +++ b/core/autocomplete/context/root-path-context/test/files/python/classes.py @@ -0,0 +1,11 @@ +class Group(BaseClass, Person): + pass + +class Group(metaclass=MetaGroup): + pass + +class Group(BaseClass[Address], Gathering[Person]): + pass + +class Group(List[Address], Person[str]): + pass \ No newline at end of file diff --git a/core/autocomplete/context/root-path-context/test/files/python/functions.py b/core/autocomplete/context/root-path-context/test/files/python/functions.py new file mode 100644 index 0000000000..74a9d8b581 --- /dev/null +++ b/core/autocomplete/context/root-path-context/test/files/python/functions.py @@ -0,0 +1,42 @@ +from typing import List, Union, TypeVar, Generic + +T = TypeVar('T') + +class Address: + pass + +class Person: + pass + +class PersonWithGeneric(Generic[T]): + pass + + +def get_address(person: Person) -> Address: + pass + +def get_group_address(people: Group[Person]) -> Group[Address]: + pass + +def log_person(person: Person) -> None: + pass + +def get_hardcoded_address() -> Address: + pass + +def log_person_or_address(value: Union[Person, Address]) -> Union[Person, Address]: + pass + +def log_person_and_address(person: Person, address: Address) -> None: + pass + +def get_address_generator(person: Person) -> Generator[Address, None, None]: + yield + + +class Group: + def log_person_and_address(self, person: Person, address: Address) -> None: + pass + +async def get_person(address: Address) -> Person: + pass \ No newline at end of file diff --git a/core/autocomplete/context/root-path-context/test/files/typescript/arrowFunctions.ts b/core/autocomplete/context/root-path-context/test/files/typescript/arrowFunctions.ts new file mode 100644 index 0000000000..aadffe0651 --- /dev/null +++ b/core/autocomplete/context/root-path-context/test/files/typescript/arrowFunctions.ts @@ -0,0 +1,29 @@ +// @ts-nocheck + +const getAddress = (person: Person): Address => { + // TODO +}; + +const logPerson = (person: Person) => { + // TODO +}; + +const getHardcodedAddress = (): Address => { + // TODO +}; + +const getAddresses = (people: Person[]): Address[] => { + // TODO +}; + +const logPersonWithAddres = (person: Person): Person => { + // TODO +}; + +const logPersonOrAddress = (person: Person | Address): Person | Address => { + // TODO +}; + +const logPersonAndAddress = (person: Person, address: Address) => { + // TODO +}; diff --git a/core/autocomplete/context/root-path-context/test/files/typescript/classMethods.ts b/core/autocomplete/context/root-path-context/test/files/typescript/classMethods.ts new file mode 100644 index 0000000000..08f7506a70 --- /dev/null +++ b/core/autocomplete/context/root-path-context/test/files/typescript/classMethods.ts @@ -0,0 +1,35 @@ +// @ts-nocheck + +class Group { + getPersonAddress(person: Person): Address { + // TODO + } + + getHardcodedAddress(): Address { + // TODO + } + + addPerson(person: Person) { + // TODO + } + + addPeople(people: Person[]) { + // TODO + } + + getAddresses(people: Person[]): Address[] { + // TODO + } + + logPersonWithAddress(person: Person): Person { + // TODO + } + + logPersonOrAddress(person: Person | Address): Person | Address { + // TODO + } + + logPersonAndAddress(person: Person, address: Address) { + // TODO + } +} diff --git a/core/autocomplete/context/root-path-context/test/files/typescript/classes.ts b/core/autocomplete/context/root-path-context/test/files/typescript/classes.ts new file mode 100644 index 0000000000..8bb8299868 --- /dev/null +++ b/core/autocomplete/context/root-path-context/test/files/typescript/classes.ts @@ -0,0 +1,9 @@ +// @ts-nocheck + +class Group extends BaseClass {} + +class Group implements FirstInterface {} + +class Group extends BaseClass implements FirstInterface, SecondInterface {} + +class Group extends BaseClass implements FirstInterface {} diff --git a/core/autocomplete/context/root-path-context/test/files/typescript/functions.ts b/core/autocomplete/context/root-path-context/test/files/typescript/functions.ts new file mode 100644 index 0000000000..bb9d39c85d --- /dev/null +++ b/core/autocomplete/context/root-path-context/test/files/typescript/functions.ts @@ -0,0 +1,33 @@ +// @ts-nocheck + +function getAddress(person: Person): Address { + // TODO +} + +function getFirstAddress(people: Person[]): Address { + // TODO +} + +function logPerson(person: Person) { + // TODO +} + +function getHardcodedAddress(): Address { + // TODO +} + +function getAddresses(people: Person[]): Address[] { + // TODO +} + +function logPersonWithAddress(person: Person): Person { + // TODO +} + +function logPersonOrAddress(person: Person | Address): Person | Address { + // TODO +} + +function logPersonAndAddress(person: Person, address: Address) { + // TODO +} diff --git a/core/autocomplete/context/root-path-context/test/files/typescript/generators.ts b/core/autocomplete/context/root-path-context/test/files/typescript/generators.ts new file mode 100644 index 0000000000..79811b4787 --- /dev/null +++ b/core/autocomplete/context/root-path-context/test/files/typescript/generators.ts @@ -0,0 +1,33 @@ +// @ts-nocheck + +function* getAddress(person: Person): Address { + // TODO +} + +function* getFirstAddress(people: Person[]): Address { + // TODO +} + +function* logPerson(person: Person) { + // TODO +} + +function* getHardcodedAddress(): Address { + // TODO +} + +function* getAddresses(people: Person[]): Address[] { + // TODO +} + +function* logPersonWithAddress(person: Person): Person { + // TODO +} + +function* logPersonOrAddress(person: Person | Address): Person | Address { + // TODO +} + +function* logPersonAndAddress(person: Person, address: Address) { + // TODO +} diff --git a/core/autocomplete/context/root-path-context/test/testCases/index.ts b/core/autocomplete/context/root-path-context/test/testCases/index.ts new file mode 100644 index 0000000000..7859fc23df --- /dev/null +++ b/core/autocomplete/context/root-path-context/test/testCases/index.ts @@ -0,0 +1,2 @@ +export * from "./python"; +export * from "./typescript"; diff --git a/core/autocomplete/context/root-path-context/test/testCases/python.ts b/core/autocomplete/context/root-path-context/test/testCases/python.ts new file mode 100644 index 0000000000..9c8d46e430 --- /dev/null +++ b/core/autocomplete/context/root-path-context/test/testCases/python.ts @@ -0,0 +1,145 @@ +export const FUNCTIONS = [ + { + nodeType: "function_definition with argument and return type", + fileName: "python/functions.py", + language: "Python", + cursorPosition: { line: 15, character: 8 }, + definitionPositions: [ + { row: 14, column: 30 }, // Person + { row: 14, column: 42 }, // Address + ], + }, + { + nodeType: + "function_definition with generic argument and generic return type", + fileName: "python/functions.py", + language: "Python", + cursorPosition: { line: 18, character: 8 }, + definitionPositions: [ + { row: 17, column: 35 }, // Group + { row: 17, column: 42 }, // Person + { row: 17, column: 53 }, // Group + { row: 17, column: 61 }, // Address + ], + }, + { + nodeType: "function_definition with single argument and None return type", + fileName: "python/functions.py", + language: "Python", + cursorPosition: { line: 21, character: 8 }, + definitionPositions: [ + { row: 20, column: 29 }, // Person + ], + }, + { + nodeType: "function_definition with no arguments and single return type", + fileName: "python/functions.py", + language: "Python", + cursorPosition: { line: 24, character: 8 }, + definitionPositions: [ + { row: 23, column: 38 }, // Address + ], + }, + { + nodeType: "function_definition with Union arguments and Union return type", + fileName: "python/functions.py", + language: "Python", + cursorPosition: { line: 27, character: 8 }, + definitionPositions: [ + { row: 26, column: 45 }, // Person + { row: 26, column: 54 }, // Address + { row: 26, column: 72 }, // Person + { row: 26, column: 81 }, // Address + ], + }, + { + nodeType: + "function_definition with multiple arguments and None return type", + fileName: "python/functions.py", + language: "Python", + cursorPosition: { line: 30, character: 8 }, + definitionPositions: [ + { row: 29, column: 41 }, // Person + { row: 29, column: 59 }, // Address + ], + }, + { + nodeType: "function_definition with one argument and Generator return type", + fileName: "python/functions.py", + language: "Python", + cursorPosition: { line: 33, character: 9 }, + definitionPositions: [ + { row: 32, column: 40 }, // Person + { row: 32, column: 62 }, // Address + ], + }, + { + nodeType: "function_definition inside a class", + fileName: "python/functions.py", + language: "Python", + cursorPosition: { line: 38, character: 12 }, + definitionPositions: [ + { row: 37, column: 51 }, // Person + { row: 37, column: 69 }, // Address + ], + }, + { + nodeType: "function_definition of an async function", + fileName: "python/functions.py", + language: "Python", + cursorPosition: { line: 41, character: 8 }, + definitionPositions: [ + { row: 40, column: 37 }, // Address + { row: 40, column: 48 }, // Person + ], + }, +]; + +export const CLASSES = [ + { + nodeType: "class_definition with multiple superclasses", + fileName: "python/classes.py", + language: "Python", + cursorPosition: { line: 1, character: 8 }, + definitionPositions: [ + { row: 0, column: 21 }, // BaseClass + { row: 0, column: 29 }, // Person + ], + }, + { + nodeType: "class_definition with multiple superclasses", + fileName: "python/classes.py", + language: "Python", + cursorPosition: { line: 4, character: 8 }, + definitionPositions: [ + { row: 3, column: 31 }, // MetaGroup + ], + }, + { + nodeType: "class_definition with generic superclasses", + fileName: "python/classes.py", + language: "Python", + cursorPosition: { line: 7, character: 8 }, + definitionPositions: [ + { row: 6, column: 21 }, // BaseClass + { row: 6, column: 29 }, // Address + { row: 6, column: 41 }, // Gathering + { row: 6, column: 48 }, // Person + ], + }, + { + nodeType: "class_definition with generic superclasses (built in types)", + fileName: "python/classes.py", + language: "Python", + cursorPosition: { line: 10, character: 8 }, + definitionPositions: [ + { row: 9, column: 24 }, // Address + { row: 9, column: 33 }, // Person + ], + }, +]; + +export const PYTHON_TEST_CASES = [ + // ...FUNCTIONS, + ...CLASSES, +]; diff --git a/core/autocomplete/context/root-path-context/test/testCases/typescript.ts b/core/autocomplete/context/root-path-context/test/testCases/typescript.ts new file mode 100644 index 0000000000..ddda2bfdec --- /dev/null +++ b/core/autocomplete/context/root-path-context/test/testCases/typescript.ts @@ -0,0 +1,397 @@ +const FUNCTIONS = [ + { + nodeType: "function_declaration with a param and a return type", + fileName: "typescript/functions.ts", + language: "TypeScript", + cursorPosition: { line: 3, character: 9 }, + definitionPositions: [ + { row: 2, column: 34 }, // Person + { row: 2, column: 44 }, // Address + ], + }, + { + nodeType: "function_declaration with array param", + fileName: "typescript/functions.ts", + language: "TypeScript", + cursorPosition: { line: 7, character: 9 }, + definitionPositions: [ + { row: 6, column: 39 }, // Person + { row: 6, column: 51 }, // Address + ], + }, + { + nodeType: "function_declaration without return type", + fileName: "typescript/functions.ts", + language: "TypeScript", + cursorPosition: { line: 11, character: 9 }, + definitionPositions: [ + { row: 10, column: 33 }, // Person + ], + }, + { + nodeType: "function_declaration without params", + fileName: "typescript/functions.ts", + language: "TypeScript", + cursorPosition: { line: 15, character: 9 }, + definitionPositions: [ + { row: 14, column: 39 }, // Person + ], + }, + { + nodeType: "function_declaration with array params and array return type", + fileName: "typescript/functions.ts", + language: "TypeScript", + cursorPosition: { line: 19, character: 9 }, + definitionPositions: [ + { row: 18, column: 36 }, // Person + { row: 18, column: 48 }, // Address + ], + }, + { + nodeType: + "function_declaration with generic params and generic return type", + fileName: "typescript/functions.ts", + language: "TypeScript", + cursorPosition: { line: 23, character: 9 }, + definitionPositions: [ + { row: 22, column: 44 }, // Person + { row: 22, column: 52 }, // Address + { row: 22, column: 62 }, // Person + { row: 22, column: 70 }, // Address + ], + }, + { + nodeType: + "function_declaration with union type params and union return type", + fileName: "typescript/functions.ts", + language: "TypeScript", + cursorPosition: { line: 27, character: 9 }, + definitionPositions: [ + { row: 26, column: 42 }, // Person + { row: 26, column: 52 }, // Address + { row: 26, column: 61 }, // Person + { row: 26, column: 71 }, // Address + ], + }, + { + nodeType: "function_declaration with two arguments", + fileName: "typescript/functions.ts", + language: "TypeScript", + cursorPosition: { line: 31, character: 9 }, + definitionPositions: [ + { row: 30, column: 43 }, // Person + { row: 30, column: 61 }, // Address + ], + }, +]; + +const GENERATORS = [ + { + nodeType: "function_declaration with a param and a return type", + fileName: "typescript/generators.ts", + language: "TypeScript", + cursorPosition: { line: 3, character: 9 }, + definitionPositions: [ + { row: 2, column: 35 }, // Person + { row: 2, column: 45 }, // Address + ], + }, + { + nodeType: "function_declaration with array param", + fileName: "typescript/generators.ts", + language: "TypeScript", + cursorPosition: { line: 7, character: 9 }, + definitionPositions: [ + { row: 6, column: 40 }, // Person + { row: 6, column: 52 }, // Address + ], + }, + { + nodeType: "function_declaration without return type", + fileName: "typescript/generators.ts", + language: "TypeScript", + cursorPosition: { line: 11, character: 9 }, + definitionPositions: [ + { row: 10, column: 34 }, // Person + ], + }, + { + nodeType: "function_declaration without params", + fileName: "typescript/generators.ts", + language: "TypeScript", + cursorPosition: { line: 15, character: 9 }, + definitionPositions: [ + { row: 14, column: 40 }, // Person + ], + }, + { + nodeType: "function_declaration with array params and array return type", + fileName: "typescript/generators.ts", + language: "TypeScript", + cursorPosition: { line: 19, character: 9 }, + definitionPositions: [ + { row: 18, column: 37 }, // Person + { row: 18, column: 49 }, // Address + ], + }, + { + nodeType: + "function_declaration with generic params and generic return type", + fileName: "typescript/generators.ts", + language: "TypeScript", + cursorPosition: { line: 23, character: 9 }, + definitionPositions: [ + { row: 22, column: 45 }, // Person + { row: 22, column: 53 }, // Address + { row: 22, column: 63 }, // Person + { row: 22, column: 71 }, // Address + ], + }, + { + nodeType: + "function_declaration with union type params and union return type", + fileName: "typescript/generators.ts", + language: "TypeScript", + cursorPosition: { line: 27, character: 9 }, + definitionPositions: [ + { row: 26, column: 43 }, // Person + { row: 26, column: 53 }, // Address + { row: 26, column: 62 }, // Person + { row: 26, column: 72 }, // Address + ], + }, + { + nodeType: "function_declaration with two arguments", + fileName: "typescript/generators.ts", + language: "TypeScript", + cursorPosition: { line: 31, character: 9 }, + definitionPositions: [ + { row: 30, column: 44 }, // Person + { row: 30, column: 62 }, // Address + ], + }, +]; + +const ARROW_FUNCTIONS = [ + { + nodeType: "arrow_function with a param and a return type", + fileName: "typescript/arrowFunctions.ts", + language: "TypeScript", + cursorPosition: { line: 3, character: 9 }, + definitionPositions: [ + { row: 2, column: 34 }, // Person + { row: 2, column: 44 }, // Address + ], + }, + { + nodeType: "arrow_function without return type", + fileName: "typescript/arrowFunctions.ts", + language: "TypeScript", + cursorPosition: { line: 7, character: 9 }, + definitionPositions: [ + { row: 6, column: 33 }, // Person + ], + }, + { + nodeType: "arrow_function without params", + fileName: "typescript/arrowFunctions.ts", + language: "TypeScript", + cursorPosition: { line: 11, character: 9 }, + definitionPositions: [ + { row: 10, column: 39 }, // Person + ], + }, + { + nodeType: "arrow_function with array params and array return type", + fileName: "typescript/arrowFunctions.ts", + language: "TypeScript", + cursorPosition: { line: 15, character: 9 }, + definitionPositions: [ + { row: 14, column: 36 }, // Person + { row: 14, column: 48 }, // Address + ], + }, + { + nodeType: "arrow_function with generic params and generic return type", + fileName: "typescript/arrowFunctions.ts", + language: "TypeScript", + cursorPosition: { line: 19, character: 9 }, + definitionPositions: [ + { row: 18, column: 43 }, // Person + { row: 18, column: 51 }, // Address + { row: 18, column: 61 }, // Person + { row: 18, column: 69 }, // Address + ], + }, + { + nodeType: "arrow_function with union type params and union return type", + fileName: "typescript/arrowFunctions.ts", + language: "TypeScript", + cursorPosition: { line: 23, character: 9 }, + definitionPositions: [ + { row: 22, column: 42 }, // Person + { row: 22, column: 52 }, // Address + { row: 22, column: 61 }, // Person + { row: 22, column: 71 }, // Address + ], + }, + { + nodeType: "arrow_function with two arguments", + fileName: "typescript/arrowFunctions.ts", + language: "TypeScript", + cursorPosition: { line: 27, character: 11 }, + definitionPositions: [ + { row: 26, column: 43 }, // Person + { row: 26, column: 61 }, // Address + ], + }, +]; + +const CLASS_METHODS = [ + { + nodeType: "method_declaration with a param and a return type", + fileName: "typescript/classMethods.ts", + language: "TypeScript", + cursorPosition: { line: 4, character: 11 }, + definitionPositions: [ + { row: 3, column: 33 }, // Person + { row: 3, column: 43 }, // Address + ], + }, + { + nodeType: "method_declaration without arguments", + fileName: "typescript/classMethods.ts", + language: "TypeScript", + cursorPosition: { line: 8, character: 11 }, + definitionPositions: [ + { row: 7, column: 32 }, // Address + ], + }, + { + nodeType: "method_declaration without return type", + fileName: "typescript/classMethods.ts", + language: "TypeScript", + cursorPosition: { line: 12, character: 11 }, + definitionPositions: [ + { row: 11, column: 26 }, // Person + ], + }, + { + nodeType: "method_declaration with array type arguments", + fileName: "typescript/classMethods.ts", + language: "TypeScript", + cursorPosition: { line: 16, character: 11 }, + definitionPositions: [ + { row: 15, column: 26 }, // Person + ], + }, + { + nodeType: + "method_declaration with array type arguments and array type return", + fileName: "typescript/classMethods.ts", + language: "TypeScript", + cursorPosition: { line: 20, character: 11 }, + definitionPositions: [ + { row: 19, column: 29 }, // Person + { row: 19, column: 41 }, // Address + ], + }, + { + nodeType: + "method_declaration with with generic params and generic return type", + fileName: "typescript/classMethods.ts", + language: "TypeScript", + cursorPosition: { line: 24, character: 11 }, + definitionPositions: [ + { row: 23, column: 37 }, // Person + { row: 23, column: 45 }, // Address + { row: 23, column: 55 }, // Person + { row: 23, column: 63 }, // Address + ], + }, + { + nodeType: "method_declaration with union type params and union return type", + fileName: "typescript/classMethods.ts", + language: "TypeScript", + cursorPosition: { line: 28, character: 11 }, + definitionPositions: [ + { row: 27, column: 35 }, // Person + { row: 27, column: 45 }, // Address + { row: 27, column: 54 }, // Person + { row: 27, column: 64 }, // Address + ], + }, + { + nodeType: "method_declaration with two arguments", + fileName: "typescript/classMethods.ts", + language: "TypeScript", + cursorPosition: { line: 32, character: 11 }, + definitionPositions: [ + { row: 31, column: 36 }, // Person + { row: 31, column: 54 }, // Address + ], + }, +]; + +const CLASS_DEFINITIONS = [ + { + nodeType: "class_declaration with base class", + fileName: "typescript/classes.ts", + language: "TypeScript", + cursorPosition: { line: 2, character: 31 }, + definitionPositions: [ + { row: 2, column: 29 }, // BaseClass + ], + }, + { + nodeType: "class_declaration with interface", + fileName: "typescript/classes.ts", + language: "TypeScript", + cursorPosition: { line: 4, character: 39 }, + definitionPositions: [ + { row: 4, column: 37 }, // FirstInterface + ], + }, + { + nodeType: "class_declaration with base class and multiple interfaces", + fileName: "typescript/classes.ts", + language: "TypeScript", + cursorPosition: { line: 6, character: 74 }, + definitionPositions: [ + { row: 6, column: 29 }, // BaseClass + { row: 6, column: 55 }, // FirstInterface + { row: 6, column: 72 }, // SecondInterface + ], + }, + { + nodeType: "class_declaration with base class and multiple interfaces", + fileName: "typescript/classes.ts", + language: "TypeScript", + cursorPosition: { line: 6, character: 74 }, + definitionPositions: [ + { row: 6, column: 29 }, // BaseClass + { row: 6, column: 55 }, // FirstInterface + { row: 6, column: 72 }, // SecondInterface + ], + }, + { + nodeType: "class_declaration with generic base class and generic interface", + fileName: "typescript/classes.ts", + language: "TypeScript", + cursorPosition: { line: 8, character: 69 }, + definitionPositions: [ + { row: 8, column: 29 }, // BaseClass + { row: 8, column: 61 }, // FirstInterface + { row: 8, column: 34 }, // User + { row: 8, column: 66 }, // User + ], + }, +]; + +export const TYPESCRIPT_TEST_CASES = [ + ...FUNCTIONS, + ...GENERATORS, + ...ARROW_FUNCTIONS, + ...CLASS_METHODS, + ...CLASS_DEFINITIONS, +]; diff --git a/core/autocomplete/context/root-path-context/test/testUtils.ts b/core/autocomplete/context/root-path-context/test/testUtils.ts new file mode 100644 index 0000000000..207b1d6d42 --- /dev/null +++ b/core/autocomplete/context/root-path-context/test/testUtils.ts @@ -0,0 +1,90 @@ +import { jest } from "@jest/globals"; +import fs from "fs"; +import path from "path"; + +import Parser from "web-tree-sitter"; +import { Position } from "../../../.."; +import { testIde } from "../../../../test/fixtures"; +import { getAst, getTreePathAtCursor } from "../../../util/ast"; +import { ImportDefinitionsService } from "../../ImportDefinitionsService"; +import { RootPathContextService } from "../RootPathContextService"; + +function splitTextAtPosition( + fileContent: string, + position: Position, +): [string, string] { + const lines = fileContent.split("\n"); + let currentPos = 0; + + // Calculate position based on the provided line and character + for (let i = 0; i < position.line; i++) { + currentPos += lines[i].length + 1; // +1 for the newline character + } + const splitPos = currentPos + position.character; + + return [fileContent.slice(0, splitPos), fileContent.slice(splitPos)]; +} + +export async function testRootPathContext( + folderName: string, + relativeFilepath: string, + position: Position, + expectedDefinitionPositions: Parser.Point[], +) { + // Create a mocked instance of RootPathContextService + const ide = testIde; + const importDefinitionsService = new ImportDefinitionsService(ide); + const service = new RootPathContextService(importDefinitionsService, ide); + + const getSnippetsMock = jest + // @ts-ignore + .spyOn(service, "getSnippets") + // @ts-ignore + .mockImplementation(async (_filepath, _endPosition) => { + return []; + }); + + // Copy the folder to the test directory + const folderPath = path.join( + __dirname, + "autocomplete", + "context", + "root-path-context", + "test", + folderName, + ); + const workspaceDir = (await ide.getWorkspaceDirs())[0]; + const testFolderPath = path.join(workspaceDir, folderName); + fs.cpSync(folderPath, testFolderPath, { + recursive: true, + force: true, + }); + + // Get results of root path context + const startPath = path.join(testFolderPath, relativeFilepath); + const [prefix, suffix] = splitTextAtPosition( + fs.readFileSync(startPath, "utf8"), + position, + ); + const fileContents = prefix + suffix; + const ast = await getAst(startPath, fileContents); + if (!ast) { + throw new Error("AST is undefined"); + } + + const treePath = await getTreePathAtCursor(ast, prefix.length); + await service.getContextForPath(startPath, treePath); + + expect(getSnippetsMock).toHaveBeenCalledTimes( + expectedDefinitionPositions.length, + ); + + expectedDefinitionPositions.forEach((position, index) => { + expect(getSnippetsMock).toHaveBeenNthCalledWith( + index + 1, + expect.any(String), // filepath argument + position, + expect.any(String), // language argument + ); + }); +} diff --git a/core/autocomplete/filter.ts b/core/autocomplete/filter.ts deleted file mode 100644 index 9fb9b7d0b6..0000000000 --- a/core/autocomplete/filter.ts +++ /dev/null @@ -1,4 +0,0 @@ -export function isOnlyPunctuationAndWhitespace(completion: string): boolean { - const punctuationAndWhitespaceRegex = /^[^\w\d\}\)\]]+$/; - return punctuationAndWhitespaceRegex.test(completion); -} diff --git a/core/autocomplete/services/BracketMatchingService.ts b/core/autocomplete/filtering/BracketMatchingService.ts similarity index 73% rename from core/autocomplete/services/BracketMatchingService.ts rename to core/autocomplete/filtering/BracketMatchingService.ts index 3734df2beb..da7d38e613 100644 --- a/core/autocomplete/services/BracketMatchingService.ts +++ b/core/autocomplete/filtering/BracketMatchingService.ts @@ -1,3 +1,13 @@ +export const BRACKETS: { [key: string]: string } = { + "(": ")", + "{": "}", + "[": "]", +}; +export const BRACKETS_REVERSE: { [key: string]: string } = { + ")": "(", + "}": "{", + "]": "[", +}; /** * We follow the policy of only completing bracket pairs that we started * But sometimes we started the pair in a previous autocomplete suggestion @@ -6,30 +16,18 @@ export class BracketMatchingService { private openingBracketsFromLastCompletion: string[] = []; private lastCompletionFile: string | undefined = undefined; - static BRACKETS: { [key: string]: string } = { "(": ")", "{": "}", "[": "]" }; - static BRACKETS_REVERSE: { [key: string]: string } = { - ")": "(", - "}": "{", - "]": "[", - }; - handleAcceptedCompletion(completion: string, filepath: string) { this.openingBracketsFromLastCompletion = []; const stack: string[] = []; for (let i = 0; i < completion.length; i++) { const char = completion[i]; - if (Object.keys(BracketMatchingService.BRACKETS).includes(char)) { + if (Object.keys(BRACKETS).includes(char)) { // It's an opening bracket stack.push(char); - } else if ( - Object.values(BracketMatchingService.BRACKETS).includes(char) - ) { + } else if (Object.values(BRACKETS).includes(char)) { // It's a closing bracket - if ( - stack.length === 0 || - BracketMatchingService.BRACKETS[stack.pop()!] !== char - ) { + if (stack.length === 0 || BRACKETS[stack.pop()!] !== char) { break; } } @@ -63,17 +61,12 @@ export class BracketMatchingService { (prefix.split("\n").pop() ?? "") + (suffix.split("\n")[0] ?? ""); for (let i = 0; i < currentLine.length; i++) { const char = currentLine[i]; - if (Object.keys(BracketMatchingService.BRACKETS).includes(char)) { + if (Object.keys(BRACKETS).includes(char)) { // It's an opening bracket stack.push(char); - } else if ( - Object.values(BracketMatchingService.BRACKETS).includes(char) - ) { + } else if (Object.values(BRACKETS).includes(char)) { // It's a closing bracket - if ( - stack.length === 0 || - BracketMatchingService.BRACKETS[stack.pop()!] !== char - ) { + if (stack.length === 0 || BRACKETS[stack.pop()!] !== char) { break; } } @@ -84,9 +77,13 @@ export class BracketMatchingService { // Add corresponding open brackets from suffix to stack // because we overwrite them and the diff is displayed, and this allows something to be edited after that for (let i = 0; i < suffix.length; i++) { - if (suffix[i] === " ") {continue;} - const openBracket = BracketMatchingService.BRACKETS_REVERSE[suffix[i]]; - if (!openBracket) {break;} + if (suffix[i] === " ") { + continue; + } + const openBracket = BRACKETS_REVERSE[suffix[i]]; + if (!openBracket) { + break; + } stack.unshift(openBracket); } @@ -111,19 +108,14 @@ export class BracketMatchingService { const allLines = all.split("\n"); for (let i = 0; i < chunk.length; i++) { const char = chunk[i]; - if (Object.values(BracketMatchingService.BRACKETS).includes(char)) { + if (Object.values(BRACKETS).includes(char)) { // It's a closing bracket - if ( - stack.length === 0 || - BracketMatchingService.BRACKETS[stack.pop()!] !== char - ) { + if (stack.length === 0 || BRACKETS[stack.pop()!] !== char) { // If the stack is empty or the top of the stack doesn't match the current closing bracket yield chunk.slice(0, i); return; // Stop the generator if the closing bracket doesn't have a matching opening bracket in the stream } - } else if ( - Object.keys(BracketMatchingService.BRACKETS).includes(char) - ) { + } else if (Object.keys(BRACKETS).includes(char)) { // It's an opening bracket stack.push(char); } diff --git a/core/autocomplete/filtering/streamTransforms/StreamTransformPipeline.ts b/core/autocomplete/filtering/streamTransforms/StreamTransformPipeline.ts new file mode 100644 index 0000000000..180c339e63 --- /dev/null +++ b/core/autocomplete/filtering/streamTransforms/StreamTransformPipeline.ts @@ -0,0 +1,98 @@ +import { streamLines } from "../../../diff/util"; +import { DEFAULT_AUTOCOMPLETE_OPTS } from "../../../util/parameters"; +import { HelperVars } from "../../util/HelperVars"; + +import { stopAtStartOf, stopAtStopTokens } from "./charStream"; +import { + avoidEmptyComments, + avoidPathLine, + noDoubleNewLine, + showWhateverWeHaveAtXMs, + skipPrefixes, + stopAtLines, + stopAtLinesExact, + stopAtRepeatingLines, + stopAtSimilarLine, + streamWithNewLines, +} from "./lineStream"; + +export class StreamTransformPipeline { + async *transform( + generator: AsyncGenerator, + prefix: string, + suffix: string, + multiline: boolean, + stopTokens: string[], + fullStop: () => void, + helper: HelperVars, + ): AsyncGenerator { + let charGenerator = generator; + + charGenerator = stopAtStopTokens(generator, stopTokens); + charGenerator = stopAtStartOf(charGenerator, suffix); + for (const charFilter of helper.lang.charFilters ?? []) { + charGenerator = charFilter({ + chars: charGenerator, + prefix, + suffix, + filepath: helper.filepath, + multiline, + }); + } + + let lineGenerator = streamLines(charGenerator); + + lineGenerator = stopAtLines(lineGenerator, fullStop); + const lineBelowCursor = this.getLineBelowCursor(helper); + if (lineBelowCursor.trim() !== "") { + lineGenerator = stopAtLinesExact(lineGenerator, fullStop, [ + lineBelowCursor, + ]); + } + lineGenerator = stopAtRepeatingLines(lineGenerator, fullStop); + lineGenerator = avoidEmptyComments( + lineGenerator, + helper.lang.singleLineComment, + ); + lineGenerator = avoidPathLine(lineGenerator, helper.lang.singleLineComment); + lineGenerator = skipPrefixes(lineGenerator); + lineGenerator = noDoubleNewLine(lineGenerator); + + for (const lineFilter of helper.lang.lineFilters ?? []) { + lineGenerator = lineFilter({ lines: lineGenerator, fullStop }); + } + + lineGenerator = stopAtSimilarLine( + lineGenerator, + this.getLineBelowCursor(helper), + fullStop, + ); + + lineGenerator = showWhateverWeHaveAtXMs( + lineGenerator, + helper.options.showWhateverWeHaveAtXMs ?? + (DEFAULT_AUTOCOMPLETE_OPTS.showWhateverWeHaveAtXMs as number), + ); + + const finalGenerator = streamWithNewLines(lineGenerator); + for await (const update of finalGenerator) { + yield update; + } + } + + private getLineBelowCursor(helper: HelperVars): string { + let lineBelowCursor = ""; + let i = 1; + while ( + lineBelowCursor.trim() === "" && + helper.pos.line + i <= helper.fileLines.length - 1 + ) { + lineBelowCursor = + helper.fileLines[ + Math.min(helper.pos.line + i, helper.fileLines.length - 1) + ]; + i++; + } + return lineBelowCursor; + } +} diff --git a/core/autocomplete/filtering/streamTransforms/charStream.test.ts b/core/autocomplete/filtering/streamTransforms/charStream.test.ts new file mode 100644 index 0000000000..4485b5a29d --- /dev/null +++ b/core/autocomplete/filtering/streamTransforms/charStream.test.ts @@ -0,0 +1,215 @@ +import { stopAtStartOf, stopAtStopTokens } from "./charStream"; + +async function* createMockStream(chunks: string[]): AsyncGenerator { + for (const chunk of chunks) { + yield chunk; + } +} + +async function streamToString(stream: AsyncGenerator): Promise { + let result = ""; + for await (const chunk of stream) { + result += chunk; + } + return result; +} + +describe("stopAtStopTokens", () => { + it("should yield characters until a stop token is encountered", async () => { + const mockStream = createMockStream(["Hello", " world", "! Stop", "here"]); + const stopTokens = ["Stop"]; + const result = stopAtStopTokens(mockStream, stopTokens); + + const output = []; + for await (const char of result) { + output.push(char); + } + + expect(output.join("")).toBe("Hello world! "); + }); + + it("should handle multiple stop tokens", async () => { + const mockStream = createMockStream([ + "This", + " is a ", + "test. END", + " of stream", + ]); + const stopTokens = ["END", "STOP", "HALT"]; + const result = stopAtStopTokens(mockStream, stopTokens); + + expect(await streamToString(result)).toBe("This is a test. "); + }); + + it("should handle stop tokens split across chunks", async () => { + const mockStream = createMockStream([ + "Hello", + " wo", + "r", + "ld! ST", + "OP now", + ]); + const stopTokens = ["STOP"]; + const result = stopAtStopTokens(mockStream, stopTokens); + + const output = []; + for await (const char of result) { + output.push(char); + } + + expect(output.join("")).toBe("Hello world! "); + }); + + it("should yield all characters if no stop token is encountered", async () => { + const mockStream = createMockStream([ + "This", + " is ", + "a complete", + " stream", + ]); + const stopTokens = ["END"]; + const result = stopAtStopTokens(mockStream, stopTokens); + + expect(await streamToString(result)).toBe("This is a complete stream"); + }); + + it("should handle empty chunks", async () => { + const mockStream = createMockStream(["Hello", "", " world", "", "! STOP"]); + const stopTokens = ["STOP"]; + const result = stopAtStopTokens(mockStream, stopTokens); + + expect(await streamToString(result)).toBe("Hello world! "); + }); + + it("should handle stop token at the beginning of the stream", async () => { + const mockStream = createMockStream(["STOP", "Hello world"]); + const stopTokens = ["STOP"]; + const result = stopAtStopTokens(mockStream, stopTokens); + + const output = []; + for await (const char of result) { + output.push(char); + } + + expect(output.join("")).toBe(""); + }); + + it("should handle stop token at the end of the stream", async () => { + const mockStream = createMockStream(["Hello world", "STOP"]); + const stopTokens = ["STOP"]; + const result = stopAtStopTokens(mockStream, stopTokens); + + expect(await streamToString(result)).toBe("Hello world"); + }); + + it("should handle multiple stop tokens of different lengths", async () => { + const mockStream = createMockStream([ + "This is a ", + "test with ", + "multiple STOP", + " tokens END", + ]); + const stopTokens = ["STOP", "END", "HALT"]; + const result = stopAtStopTokens(mockStream, stopTokens); + + expect(await streamToString(result)).toBe("This is a test with multiple "); + }); + + it("should handle an empty stream", async () => { + const mockStream = createMockStream([]); + const stopTokens = ["STOP"]; + const result = stopAtStopTokens(mockStream, stopTokens); + + expect(await streamToString(result)).toBe(""); + }); + + it("should handle an empty stop tokens array", async () => { + const mockStream = createMockStream(["Hello", " world!"]); + const stopTokens: string[] = []; + const result = stopAtStopTokens(mockStream, stopTokens); + + const output = []; + for await (const char of result) { + output.push(char); + } + + expect(output.join("")).toBe("Hello world!"); + }); + + it("should handle stop token when remaining buffer is smaller than maximum stop token length", async () => { + const mockStream = createMockStream(["Hello world!STOP"]); + const stopTokens: string[] = [ + "STOP", + "STOP_TOKEN_THAT_IS_LARGER_THAN_BUFFER", + ]; + const result = stopAtStopTokens(mockStream, stopTokens); + + expect(await streamToString(result)).toBe("Hello world!"); + }); +}); + +describe("stopAtStartOf", () => { + const sampleCode = ` { + method: "GET", + headers: { + "Content-Type": "application/json", + Authorization: \`Bearer \${this.workOsAccessToken}\`, + }, + }, + ); + const data = await response.json(); + return data.items; + } + + async getContextItems( + query: string, + extras: ContextProviderExtras, + ): Promise { + const response = await extras.fetch( + new URL( + \`/proxy/context/\${this.options.id}/retrieve\`, + controlPlaneEnv.CONTROL_PLANE_URL, + ), +`; + + /* Some LLMs, such as Codestral, repeat the suffix of the query. To test our filtering, we cut the sample code at random positions, remove a part of the input +and construct a response, containing the removed part and the suffix. The goal of the stopAtStartOf() method is to detect the start of the suffix in the response */ + it("should stop if the start of the suffix is reached", async () => { + const suffix = ` + const data = await response.json(); + return data.items; +}`; + const mockStream = createMockStream(sampleCode.split(/(?! )/g)); + const result = stopAtStartOf(mockStream, suffix); + + const resultStr = await streamToString(result); + expect(resultStr).toBe(` { + method: "GET", + headers: { + "Content-Type": "application/json", + Authorization: \`Bearer \${this.workOsAccessToken}\`, + }, + }, + ); + `); + }); + it("should stop if the start of the suffix is reached, even if the suffix has a prefix", async () => { + const suffix = ` + xxxconst data = await response.json(); + return data.items; +}`; + const mockStream = createMockStream(sampleCode.split(/(?! )/g)); + const result = stopAtStartOf(mockStream, suffix); + + const resultStr = await streamToString(result); + expect(resultStr).toBe(` { + method: "GET", + headers: { + "Content-Type": "application/json", + Authorization: \`Bearer \${this.workOsAccessToken}\`, + }, + }, + ); + `); + }); +}); diff --git a/core/autocomplete/streamTransforms/charStream.ts b/core/autocomplete/filtering/streamTransforms/charStream.ts similarity index 68% rename from core/autocomplete/streamTransforms/charStream.ts rename to core/autocomplete/filtering/streamTransforms/charStream.ts index 79fc33a0d8..9177a97163 100644 --- a/core/autocomplete/streamTransforms/charStream.ts +++ b/core/autocomplete/filtering/streamTransforms/charStream.ts @@ -70,7 +70,8 @@ export async function* noFirstCharNewline(stream: AsyncGenerator) { * 2. Otherwise, buffers incoming chunks and checks for stop tokens. * 3. Yields characters one by one if no stop token is found at the start of the buffer. * 4. Stops yielding and returns if a stop token is encountered. - * 5. After the stream ends, yields any remaining buffered characters. + * 5. After the stream ends, filters encountered stop tokens in remaining buffer. + * 6. Yields any remaining buffered characters. */ export async function* stopAtStopTokens( stream: AsyncGenerator, @@ -106,9 +107,57 @@ export async function* stopAtStopTokens( } } } + // Filter out the possible stop tokens from remaining buffer + stopTokens.forEach((token) => { + buffer = buffer.replace(token, ""); + }); // Yield any remaining characters in the buffer for (const char of buffer) { yield char; } } + +/** + * Asynchronously yields characters from the input stream. + * Stops if the beginning of the suffix is detected in the stream. + */ +export async function* stopAtStartOf( + stream: AsyncGenerator, + suffix: string, + sequenceLength: number = 20, +): AsyncGenerator { + if (suffix.length < sequenceLength) { + for await (const chunk of stream) { + yield chunk; + } + return; + } + // We use sequenceLength * 1.5 as a heuristic to make sure we don't miss the sequence if the + // stream is not perfectly aligned with the sequence (small whitespace differences etc). + const targetPart = suffix + .trimStart() + .slice(0, Math.floor(sequenceLength * 1.5)); + + let buffer = ""; + + for await (const chunk of stream) { + buffer += chunk; + + // Check if the targetPart contains contains the buffer at any point + if (buffer.length >= sequenceLength && targetPart.includes(buffer)) { + return; // Stop processing when the sequence is found + } + + // Yield chunk by chunk, ensuring not to exceed sequenceLength in the buffer + while (buffer.length > sequenceLength) { + yield buffer[0]; + buffer = buffer.slice(1); + } + } + + // Yield the remaining buffer if it is not contained in the `targetPart` + if (buffer.length > 0) { + yield buffer; + } +} diff --git a/core/autocomplete/lineStream.test.ts b/core/autocomplete/filtering/streamTransforms/lineStream.test.ts similarity index 93% rename from core/autocomplete/lineStream.test.ts rename to core/autocomplete/filtering/streamTransforms/lineStream.test.ts index ace94eaa42..5cb05d2c3a 100644 --- a/core/autocomplete/lineStream.test.ts +++ b/core/autocomplete/filtering/streamTransforms/lineStream.test.ts @@ -1,5 +1,6 @@ import { jest } from "@jest/globals"; -import * as lineStream from "./streamTransforms/lineStream"; + +import * as lineStream from "./lineStream"; describe("lineStream", () => { let mockFullStop: jest.Mock; @@ -31,21 +32,34 @@ describe("lineStream", () => { }); describe("avoidPathLine", () => { - it("should filter out path lines and empty comments", async () => { + it("should filter out path lines", async () => { const linesGenerator = await getLineGenerator([ "// Path: src/index.ts", "const x = 5;", "//", "console.log(x);", ]); + const result = lineStream.avoidPathLine(linesGenerator, "//"); + const filteredLines = await getFilteredLines(result); + expect(filteredLines).toEqual(["const x = 5;", "//", "console.log(x);"]); + }); + }); - const result = lineStream.avoidPathLineAndEmptyComments( - linesGenerator, + describe("avoidEmptyComments", () => { + it("should filter out empty comments", async () => { + const linesGenerator = await getLineGenerator([ + "// Path: src/index.ts", + "const x = 5;", "//", - ); + "console.log(x);", + ]); + const result = lineStream.avoidEmptyComments(linesGenerator, "//"); const filteredLines = await getFilteredLines(result); - - expect(filteredLines).toEqual(["const x = 5;", "console.log(x);"]); + expect(filteredLines).toEqual([ + "// Path: src/index.ts", + "const x = 5;", + "console.log(x);", + ]); }); }); @@ -84,7 +98,7 @@ describe("lineStream", () => { describe("stopAtSimilarLine", () => { it("should stop at the exact same line", async () => { - const lineToTest = "const x = 6;"; + const lineToTest = "const x = 6"; const linesGenerator = await getLineGenerator([ "console.log();", "const y = () => {};", diff --git a/core/autocomplete/streamTransforms/lineStream.ts b/core/autocomplete/filtering/streamTransforms/lineStream.ts similarity index 85% rename from core/autocomplete/streamTransforms/lineStream.ts rename to core/autocomplete/filtering/streamTransforms/lineStream.ts index 5990bbf3cd..851dea88ab 100644 --- a/core/autocomplete/streamTransforms/lineStream.ts +++ b/core/autocomplete/filtering/streamTransforms/lineStream.ts @@ -1,12 +1,21 @@ import { distance } from "fastest-levenshtein"; -import { DiffLine } from "../.."; -import { LineStream } from "../../diff/util"; + +import { DiffLine } from "../../.."; +import { LineStream } from "../../../diff/util"; export type LineFilter = (args: { lines: LineStream; fullStop: () => void; }) => LineStream; +export type CharacterFilter = (args: { + chars: AsyncGenerator; + prefix: string; + suffix: string; + filepath: string; + multiline: boolean; +}) => AsyncGenerator; + function isBracketEnding(line: string): boolean { return line .trim() @@ -14,14 +23,6 @@ function isBracketEnding(line: string): boolean { .some((char) => BRACKET_ENDING_CHARS.includes(char)); } -function commonPrefixLength(a: string, b: string): number { - let i = 0; - while (i < a.length && i < b.length && a[i] === b[i]) { - i++; - } - return i; -} - function isEnglishFirstLine(line: string) { line = line.trim().toLowerCase(); @@ -81,6 +82,7 @@ export const LINES_TO_REMOVE_BEFORE_START = [ "", "[CODE]", "", + "{{FILL_HERE}}", ]; export const ENGLISH_START_PHRASES = [ @@ -122,27 +124,45 @@ export async function* noTopLevelKeywordsMidline( } /** - * Filters out unwanted lines from a LineStream, specifically those starting with '// Path: ' or empty comments. + * Filters out lines starting with '// Path: ' from a LineStream. * * @param {LineStream} stream - The input stream of lines to filter. * @param {string} comment - The comment syntax to filter (e.g., '//' for JavaScript-style comments). - * @yields {string} The filtered lines, excluding unwanted path lines and empty comments. + * @yields {string} The filtered lines, excluding unwanted path lines. */ -export async function* avoidPathLineAndEmptyComments( +export async function* avoidPathLine( stream: LineStream, - comment: string, + comment?: string, ): LineStream { // Snippets are inserted as comments with a line at the start '// Path: '. // Sometimes the model with copy this pattern, which is unwanted for await (const line of stream) { - // Also filter lines that are empty comments - if (line.startsWith(`${comment} Path: `) || line.trim() === comment) { + if (line.startsWith(`${comment} Path: `)) { continue; } yield line; } } +/** + * Filters out empty comment lines from a LineStream. + * + * @param {LineStream} stream - The input stream of lines to filter. + * @param {string} comment - The comment syntax to filter (e.g., '//' for JavaScript-style comments). + * @yields {string} The filtered lines, excluding empty comments. + */ +export async function* avoidEmptyComments( + stream: LineStream, + comment?: string, +): LineStream { + // Filter lines that are empty comments + for await (const line of stream) { + if (!comment || line.trim() !== comment) { + yield line; + } + } +} + /** * Transforms a LineStream by adding newline characters between lines. * @@ -168,9 +188,7 @@ export async function* streamWithNewLines(stream: LineStream): LineStream { * @returns {boolean} True if the lines are considered repeated, false otherwise. * * @description - * This function checks if two lines are repeated or very similar based on two criteria: - * 1. They have a common prefix longer than 12 characters. - * 2. The Levenshtein distance between them is less than 10% of the length of the second line. + * This function checks if the Levenshtein distance between them is less than 10% of the length of the second line. * Lines shorter than 5 characters are never considered repeated. */ export function lineIsRepeated(a: string, b: string): boolean { @@ -180,10 +198,7 @@ export function lineIsRepeated(a: string, b: string): boolean { const aTrim = a.trim(); const bTrim = b.trim(); - return ( - commonPrefixLength(aTrim, bTrim) > 12 || - distance(aTrim, bTrim) / bTrim.length < 0.1 - ); + return distance(aTrim, bTrim) / bTrim.length < 0.1; } /** @@ -210,9 +225,9 @@ export async function* stopAtSimilarLine( const lineIsBracketEnding = isBracketEnding(trimmedLine); for await (const nextLine of stream) { - if (nextLine === line) { - fullStop(); - break; + if (trimmedLine === "") { + yield nextLine; + continue; } if (lineIsBracketEnding && trimmedLine.trim() === nextLine.trim()) { @@ -220,6 +235,11 @@ export async function* stopAtSimilarLine( continue; } + if (nextLine === line) { + fullStop(); + break; + } + if (lineIsRepeated(nextLine, trimmedLine)) { fullStop(); break; @@ -249,6 +269,20 @@ export async function* stopAtLines( } } +export async function* stopAtLinesExact( + stream: LineStream, + fullStop: () => void, + linesToStopAt: string[], +): LineStream { + for await (const line of stream) { + if (linesToStopAt.some((stopAt) => line === stopAt)) { + fullStop(); + break; + } + yield line; + } +} + /** * Filters a LineStream, skipping specified prefixes on the first line. * @param {LineStream} lines - The input stream of lines. @@ -510,3 +544,38 @@ export async function* logLines( } console.log(`${prefix}:\n${linesToLog.join("\n")}\n\n`); } + +export async function* showWhateverWeHaveAtXMs( + lines: LineStream, + ms: number, +): LineStream { + const startTime = Date.now(); + let firstNonWhitespaceLineYielded = false; + + for await (const line of lines) { + yield line; + + if (!firstNonWhitespaceLineYielded && line.trim() !== "") { + firstNonWhitespaceLineYielded = true; + } + + const isTakingTooLong = Date.now() - startTime > ms; + if (isTakingTooLong && firstNonWhitespaceLineYielded) { + break; + } + } +} + +export async function* noDoubleNewLine(lines: LineStream): LineStream { + let isFirstLine = true; + + for await (const line of lines) { + if (line.trim() === "" && !isFirstLine) { + return; + } + + isFirstLine = false; + + yield line; + } +} diff --git a/core/autocomplete/filtering/test/NEGATIVE_TEST_CASES/QWEN_JSON.txt b/core/autocomplete/filtering/test/NEGATIVE_TEST_CASES/QWEN_JSON.txt new file mode 100644 index 0000000000..ff0ba3cb39 --- /dev/null +++ b/core/autocomplete/filtering/test/NEGATIVE_TEST_CASES/QWEN_JSON.txt @@ -0,0 +1,60 @@ +##### Prompt ##### +{ + "active": true, + "department": "Product Development", + "location": { + "country": "USA", + "state": "California", + "city": "San BERNARDINO", + "coordinates": { + + } + }, + "employees": [ + { + "name": "John Doe", + "age": 30, + "position": "Developer", + "skills": ["JavaScript", "React", "Node.js"], + "remote": false, + "salary": { + "currency": "USD", + "amount": 95000 + } + }, + { + "name": "Jane Smith", + "age": 25, + "position": "Designer", + "skills": ["Photoshop", "Illustrator"], + "remote": true, + "salary": { + "currency": "USD", + "amount": 70000 + } + }, + { + "name": "Emily Johnson", + "age": 35, + "position": "Manager", + "teamSize": 8, + "remote": true, + "skills": ["Leadership", "Project Management"],========================================================================== +========================================================================== +Completion: + + "latitude": 34.10834, + "longitude": -117.28977 + } + }, + "employeeCount": 2, + "averageAge": 30, + "remoteFriendly": true, + "salaryRange": { + "min": 70000, + "max": 95000, + "currency": "USD" + }, + "skills": { + "required": ["JavaScript", "React", "Node.js", "Leadership", "Project Management"], + "optional": ["Photoshop", "Illustrator"] \ No newline at end of file diff --git a/core/autocomplete/filtering/test/NEGATIVE_TEST_CASES/QWEN_TYPESCRIPT.txt b/core/autocomplete/filtering/test/NEGATIVE_TEST_CASES/QWEN_TYPESCRIPT.txt new file mode 100644 index 0000000000..241d141f28 --- /dev/null +++ b/core/autocomplete/filtering/test/NEGATIVE_TEST_CASES/QWEN_TYPESCRIPT.txt @@ -0,0 +1,128 @@ +##### Prompt ##### + }`, + }, + { + description: "Should autocomplete Vue computed property", + filename: "UserComponent.vue", + input: ` + + User Full Name: {{ fullName }} + + + + +`, + llmOutput: `() { + return this.firstName + ' ' + this.lastName; + }`, + expectedCompletion: `() { + return this.firstName + ' ' + this.lastName; + }`, + }, + { + description: "Should autocomplete Vue method using props", + filename: "TodoItem.vue", + input: ` + + {{ title }} + Complete + + + + +`, + llmOutput: `this.completed`, + expectedCompletion: `this.completed`, + }, + { + description: "Should autocomplete Svelte reactive statement", + filename: "Counter.svelte", + input: ` + + + + Clicked {count} times + +`, + llmOutput: `doubledCount = count * 2`, + expectedCompletion: `doubledCount = count * 2`, + }, + + { + description: "Should autocomplete Svelte component inside HTML", + filename: "NestedComponent.svelte", + input: ` + + + + Hello Svelte + /> + +`, + llmOutput: `name="World"`, + expectedCompletion: `name="World"`, + }, + + { + description: "Should handle autocomplete in Svelte each block", + filename: "List.svelte", + input: ` + + + + {#each items as item} + {item} + {/each<|fim|> + +`, + llmOutput: `}`, + expectedCompletion: `}`, + }, + +]; +========================================================================== +========================================================================== +Completion: + + + +export default { + components: { + ChildComponent, + }, \ No newline at end of file diff --git a/core/autocomplete/filtering/test/NEGATIVE_TEST_CASES/STARCODER_JSON.txt b/core/autocomplete/filtering/test/NEGATIVE_TEST_CASES/STARCODER_JSON.txt new file mode 100644 index 0000000000..a50bd2fb4c --- /dev/null +++ b/core/autocomplete/filtering/test/NEGATIVE_TEST_CASES/STARCODER_JSON.txt @@ -0,0 +1,20 @@ +##### Prompt ##### +{ + "employees": [ + { "name": "John Doe", "age": 30, "position": "Developer" }, + { "name": "Jane Smith", "age": 25, "position": "Designer" }, + { "name": "Emily Johnson", "age": 35, "position": "Manager" } + ], + "active": true +} +========================================================================== +========================================================================== +Completion: + + +} + +{ + "employees": [ + { "name": "John Doe", "age": 30 }, + { "name": "Jane Smith", "age": 25 } \ No newline at end of file diff --git a/core/autocomplete/filtering/test/NEGATIVE_TEST_CASES/STARCODER_PYTHON.TXT b/core/autocomplete/filtering/test/NEGATIVE_TEST_CASES/STARCODER_PYTHON.TXT new file mode 100644 index 0000000000..c4aaf43ec1 --- /dev/null +++ b/core/autocomplete/filtering/test/NEGATIVE_TEST_CASES/STARCODER_PYTHON.TXT @@ -0,0 +1,29 @@ +##### Prompt ##### +class Calculator: + def __init__(self): + self.result = 0 + + def add(self, number): + self.result += number + return self + + def divid + + def subtract(self, number): + self.result -= number + return self + + def reset(self): + self.result = 0 + return self + + def get_result(self): + return self.result + + +========================================================================== +========================================================================== +Completion: + +self.result /= number + return self \ No newline at end of file diff --git a/core/autocomplete/filtering/test/NEGATIVE_TEST_CASES/STARCODER_RUBY.TXT b/core/autocomplete/filtering/test/NEGATIVE_TEST_CASES/STARCODER_RUBY.TXT new file mode 100644 index 0000000000..df8c2011fb --- /dev/null +++ b/core/autocomplete/filtering/test/NEGATIVE_TEST_CASES/STARCODER_RUBY.TXT @@ -0,0 +1,41 @@ +##### Prompt ##### +class Calculator + attr_accessor :result + + def initialize + @result = 0 + end + + def add(number) + @result += number + self + end + + def subtract(number) + @result -= number + self + end + + def power(ne + + def get_result + @result + end + + def reset + @result = 0 + self + end +end + +calc = Calculator.new +calc.add(5).subtract(3) +puts "Result: #{calc.get_result}" +========================================================================== +========================================================================== +Completion: + + + @result ** number + self + end \ No newline at end of file diff --git a/core/autocomplete/filtering/test/NEGATIVE_TEST_CASES/STARCODER_RUST.TXT b/core/autocomplete/filtering/test/NEGATIVE_TEST_CASES/STARCODER_RUST.TXT new file mode 100644 index 0000000000..582ae5f364 --- /dev/null +++ b/core/autocomplete/filtering/test/NEGATIVE_TEST_CASES/STARCODER_RUST.TXT @@ -0,0 +1,73 @@ +##### Prompt ##### +use std::io; + +fn main() { + println!("Welcome to the Calculator!"); + + loop { + println!("Please enter an operator (+, -, *, /) or 'q' to quit:"); + let operator = read_input(); + + if operator == "q" { + break; + } + + println!("Please enter the first number:"); + let num1 = read_input().parse::().unwrap(); + + println!("Please enter the second number:"); + let num2 = read_input().parse::().unwrap(); + + let result = match operator.as_str() { + "+" => num1 + num2, + "-" => num1 - num2, + "*" => num1 * num2, + "/" => num1 / num2, + + _ => { + println!("Invalid operator. Please try again."); + continue; + } + }; + + println!("Result: {}", result); + } +} + +fn read_input() -> String { + let mut input = String::new(); + io::stdin() + .read_line(&mut input) + .expect("Failed to read input"); + input.trim().to_string() +} +========================================================================== +========================================================================== +Completion: + + => { + println!("Invalid operator. Please try again."); + continue; + } + }; + + println!("Result: {}", result); + } + + println!("Goodbye!"); +} + + +fn main2() { + println!("Welcome to the Calculator!"); + + loop { + println!("Please enter an operator (+, -, *, /) or 'q' to quit:"); + let operator = read_input(); + + if operator == "q" { + break; + } + + match operator.as_str() { + "+" | "-" | "*" | "/" => (), \ No newline at end of file diff --git a/core/autocomplete/filtering/test/filter.test.ts b/core/autocomplete/filtering/test/filter.test.ts new file mode 100644 index 0000000000..594e061ad6 --- /dev/null +++ b/core/autocomplete/filtering/test/filter.test.ts @@ -0,0 +1,48 @@ +import { setUpTestDir, tearDownTestDir } from "../../../test/testDir"; + +import { TEST_CASES_WITH_DIFF, TEST_CASES_WITHOUT_DIFF } from "./testCases"; +import { + AutocompleteFileringTestInput, + testAutocompleteFiltering, +} from "./util"; + +const filterTestCases = (tests: AutocompleteFileringTestInput[]) => { + if (tests.some((test) => test.options?.only)) { + return tests.filter((test) => test.options?.only); + } + + return tests; +}; + +describe("Autocomplete filtering tests", () => { + beforeAll(async () => { + tearDownTestDir(); + setUpTestDir(); + }); + + afterAll(async () => { + tearDownTestDir(); + }); + + beforeEach(async () => { + await new Promise((resolve) => setTimeout(resolve, 50)); + }); + + describe("Should return unmodified LLM output", () => { + it.each(filterTestCases(TEST_CASES_WITHOUT_DIFF))( + "$description", + async (testCase) => { + await testAutocompleteFiltering(testCase); + }, + ); + }); + + describe("Should return modified LLM output", () => { + it.each(filterTestCases(TEST_CASES_WITH_DIFF))( + "$description", + async (testCase) => { + await testAutocompleteFiltering(testCase); + }, + ); + }); +}); diff --git a/core/autocomplete/filtering/test/testCases.ts b/core/autocomplete/filtering/test/testCases.ts new file mode 100644 index 0000000000..f83de16073 --- /dev/null +++ b/core/autocomplete/filtering/test/testCases.ts @@ -0,0 +1,2200 @@ +import { dedent } from "../../../util"; + +import { AutocompleteFileringTestInput } from "./util"; + +export const TEST_CASES_WITH_DIFF: AutocompleteFileringTestInput[] = [ + { + description: "Should handle python multi-line string", + filename: "test.py", + input: `def create_greeting(name): + greeting = """Hello, """ + name + """! +Welcome to our community. We hope you have a great time here. +If you have any questions, feel free to reach out.""" + return greeting + +message = create_greeting("Alice") +print(message) + +multi_line_message = """<|fim|> +print(multi_line_message) +`, + llmOutput: `This is a multi-line message. +It continues across multiple lines, +which allows for easy reading and formatting. +""" +`, + expectedCompletion: `This is a multi-line message. +It continues across multiple lines, +which allows for easy reading and formatting. +"""`, + }, + { + description: "Should autocomplete Rust match arms", + filename: "main.rs", + input: ` +fn get_status_code_description(code: u16) -> &'static str { + match code { + 200 => "OK", + 404 => "Not Found", + 500 => "Internal Server Error", + <|fim|> + } +} +`, + llmOutput: `403 => "Forbidden", + 401 => "Unauthorized", + _ => "Unknown Status", +`, + expectedCompletion: `403 => "Forbidden", + 401 => "Unauthorized", + _ => "Unknown Status",`, + }, + { + description: "Should complete a Markdown code block", + filename: "test.md", + input: ` +Here is a sample JavaScript function: + +\`\`\`javascript +function sayHello() { + console.log("Hello, <|fim|> +} +\`\`\` +`, + llmOutput: `world!"); +`, + expectedCompletion: 'world!");', + }, + { + description: "Should autocomplete Java when inside a block", + filename: "Main.java", + input: ` +public class Main { + public static void main(String[] args) { + for (int i = 0; i < 10; i++) { + if (i % 2 == 0) { + System.out.println("Even: " + i); + } else { +<|fim|> + } + } + } +}`, + llmOutput: ` + System.out.println("Odd: " + i); +`, + expectedCompletion: ` + System.out.println("Odd: " + i);`, + }, + { + description: + "Should autocomplete a Markdown heading and preserve formatting", + filename: "test.md", + input: `# My Document + +## Introduction +This is a sample document for testing. + +## <|fim|> +### Conclusion +Thank you for reading. +`, + llmOutput: `Features +Here is a list of features: +- Feature 1 +- Feature 2 +`, + expectedCompletion: `Features +Here is a list of features: +- Feature 1 +- Feature 2`, + }, + { + // options: { only: true }, + description: "Should autocomplete a Java method within a class", + filename: "Calculator.java", + input: ` +public class Calculator { + private double result; + + public Calculator() { + this.result = 0.0; + } + + public void add(double number) { + result += number; +<|fim|>`, + llmOutput: ` + } + + public void subtract(double number) { + result -= number; + } +`, + expectedCompletion: ` + }`, + }, + { + description: "Should filter out consecutive, repeated YAML keys", + filename: "test.yaml", + input: ` + version: '3' + services: + db: + image: postgres + environment: + POSTGRES_USER: user + POSTGRES_PASSWORD: pass<|fim|> + `, + llmOutput: ` + POSTGRES_DB: mydb + POSTGRES_DB: mydb + `, + expectedCompletion: ` + POSTGRES_DB: mydb`, + }, + { + description: "Should enforce bracket matching in JSON files", + filename: "test.json", + input: `{ + "active": true, + "department": "Product Development", + "location": { + "country": "USA", + "state": "California", + "city": "San BERNARDINO", + "coordinates": { + <|fim|> + } + }, + "employees": [ + { + "name": "John Doe", + "age": 30, + "position": "Developer", + "skills": ["JavaScript", "React", "Node.js"], + "remote": false, + "salary": { + "currency": "USD", + "amount": 95000 + } + }, + { + "name": "Jane Smith", + "age": 25, + "position": "Designer", + "skills": ["Photoshop", "Illustrator"], + "remote": true, + "salary": { + "currency": "USD", + "amount": 70000 + } + }, + { + "name": "Emily Johnson", + "age": 35, + "position": "Manager", + "teamSize": 8, + "remote": true, + "skills": ["Leadership", "Project Management"],`, + llmOutput: `"latitude": 34.10834, + "longitude": -117.28977 + } + }, + "employeeCount": 2, + "averageAge": 30, + "remoteFriendly": true, + "salaryRange": { + "min": 70000, + "max": 95000, + "currency": "USD" + }, + "skills": { + "required": ["JavaScript", "React", "Node.js", "Leadership", "Project Management"], + "optional": ["Photoshop", "Illustrator"]`, + expectedCompletion: `"latitude": 34.10834, + "longitude": -117.28977`, + }, + { + description: + "Should return nothing when output is duplicated lines in TypeScript", + filename: "file.ts", + input: ` +async getContextForPath( + filepath: string, + astPath: AstPatt, + language: LanguageName, + options: ContextOptions = {}, +<|fim|> + ): Promise { + const snippets: AutocompleteCodeSnippet[] = []; + let parentKey = filepath; +`, + llmOutput: ` ): Promise { + const snippets: AutocompleteCodeSnippet[] = []; + `, + expectedCompletion: undefined, + }, + { + description: + "Should return partial result when output is duplicated lines in TypeScript", + filename: "file.ts", + input: ` +async getContextForPath( + filepath: string, + astPath: AstPatt, + language: LanguageName, + options: ContextOptions = {}, +<|fim|> + ): Promise { + const snippets: AutocompleteCodeSnippet[] = []; + let parentKey = filepath; +`, + llmOutput: `console.log('TEST'); + ): Promise { + const snippets: AutocompleteCodeSnippet[] = []; + `, + expectedCompletion: `console.log('TEST');`, + }, + { + description: "Should autocomplete React effect hook", + input: `import React, { useState, useEffect } from "react"; + +export const Timer = () => { + const [seconds, setSeconds] = useState(0); + + useEffect(() => { + const interval = setInterval(() => { + setSeconds(seconds + 1); + }, 1000); + + <|fim|>; + + return () => clearInterval(interval); + }, [seconds]); + + return ( + + {seconds} seconds have passed. + + ); +};`, + llmOutput: "return () => clearInterval(interval);", + expectedCompletion: undefined, + filename: "Timer.tsx", + }, + { + description: "Should autocomplete simple return statement in TypeScript", + filename: "file.ts", + input: ` + multiply(number) { + this.result *= number; + return <|fim|> + } + + divide(number) { + if (number === 0) { + throw new Error("Cannot divide by zero"); + } + this.result /= number; + return this; + } +`, + llmOutput: ` this;`, + expectedCompletion: `this;`, + }, + { + description: "Should complete YAML list item and preserve structure", + filename: "test.yaml", + input: ` + services: + - name: web + image: nginx + - name: app<|fim|> + volumes: + - volume1 + - volume2 + `, + llmOutput: ` + image: node + `, + expectedCompletion: ` + image: node`, + }, + { + description: + "Should complete YAML key-value pair inside a nested structure", + filename: "test.yaml", + input: ` + version: '3' + services: + db: + image: postgres + environment: + POSTGRES_USER: user + POSTGRES_PASSWORD: pass + POSTGRES_DB: mydb<|fim|> + `, + llmOutput: ` + PGDATA: /var/lib/postgresql/data/pgdata + `, + expectedCompletion: ` + PGDATA: /var/lib/postgresql/data/pgdata`, + }, + { + description: "Should complete YAML block within an existing block", + filename: "test.yaml", + input: ` + pipelines: + branches: + master: + - step: + name: Build and Test + script: + - npm install + - npm run test + - step: + name: Deploy<|fim|> + `, + llmOutput: ` + script: + - npm run deploy + `, + expectedCompletion: ` + script: + - npm run deploy`, + }, + { + description: + "Should autocomplete SQL query with subquery and alias in SELECT clause", + filename: "complex_query.sql", + input: `SELECT u.id, + u.name, + (SELECT COUNT(*) FROM orders o WHERE o.user_id = u.id) AS order_count + FROM users u + WHERE u.active = 1 + <|fim|>`, + llmOutput: + " AND EXISTS (SELECT 1 FROM transactions t WHERE t.user_id = u.id AND t.amount > 100)", + expectedCompletion: + "AND EXISTS (SELECT 1 FROM transactions t WHERE t.user_id = u.id AND t.amount > 100)", + }, +]; + +export const TEST_CASES_WITHOUT_DIFF: AutocompleteFileringTestInput[] = [ + { + description: "should pass", + filename: "test.js", + input: "console.log('Hello <|fim|>!');", + llmOutput: "World", + expectedCompletion: "World", + }, + { + description: + "Should preserve closing brackets when the opening bracket is not a part of the completion.", + filename: "test.js", + input: dedent` + class Calculator { + constructor() { + this.result = 0; + } + + add(number) { + this.result += number; + return this; + } + + subtract(number) { + this.result -= number; + return this; + } + + multiply(number) { + this.result *= number; + return this; + } + + divide(number) { + <|fim|> + + getResult() { + return this.result; + } + + reset() { + this.result = 0; + return this; + } + } + `, + llmOutput: dedent`if (number === 0) { + throw new Error("Cannot divide by zero"); + } + this.result /= number; + return this; + }`, + expectedCompletion: dedent`if (number === 0) { + throw new Error("Cannot divide by zero"); + } + this.result /= number; + return this; + }`, + }, + { + description: "Should multiline-autocomplete CSS blocks", + input: `body { + font-family: Arial, sans-serif; + font-size: 16px; + color: #333333; +} + +h1 { + font-size: 24px; + font-weight: bold; + color: #000000; +} + +h4<|fim|> + +a { + text-decoration: none; + color: #007bff; +} + +.container { + width: 100%; + max-width: 1200px; + margin: 0 auto; + padding: 20px; +} + +.button { + display: inline-block; + padding: 10px 20px; + background-color: #007bff; + color: #ffffff;`, + llmOutput: ` { + font-size: 18px; + font-weight: bold; + color: #000000; +}`, + expectedCompletion: ` { + font-size: 18px; + font-weight: bold; + color: #000000; +}`, + filename: "test.css", + }, + { + description: "Should complete CSS rules inside a nested selector", + filename: "styles.css", + input: ` +.container { + display: flex; + justify-content: center; + align-items: center; + width: 100%; + height: 100vh; + + .inner { + <|fim|> + } +} +`, + llmOutput: `width: 50%; + height: 50%; + background-color: #f0f0f0;`, + expectedCompletion: `width: 50%; + height: 50%; + background-color: #f0f0f0;`, + }, + + { + description: + "Should complete a CSS rule when the property is partially typed", + filename: "styles.css", + input: ` +button { + border: 2px solid #000; + border-radius<|fim|> +}`, + llmOutput: ": 5px;", + expectedCompletion: ": 5px;", + }, + + { + description: "Should handle CSS autocomplete with a single bracket present", + filename: "styles.css", + input: ` +.card { + box-shadow: 0 4px 8px rgba(0,0,0,0.2); + transition: 0.3s; + padding: 16px; + border-bottom-left-radius: <|fim|>px; + border-bottom-right-radius: 8px; +} +`, + llmOutput: "8", + expectedCompletion: "8", + }, + + { + description: "Should autocomplete CSS pseudoclass", + filename: "pseudoClass.css", + input: ` +input:focus { + outline: none; + border: 2px solid <|fim|>; +} +`, + llmOutput: "#4CAF50;", + expectedCompletion: "#4CAF50;", + }, + + { + description: "Should handle CSS variable syntax", + filename: "variables.css", + input: ` +:root { + --primary-color: #3498db; + --padding: 10px; +} + +.section { + background-color: var(<|fim|>); + padding: var(--padding); +} +`, + llmOutput: "--primary-color", + expectedCompletion: "--primary-color", + }, + { + description: "Should complete CSS grid template columns", + filename: "grid.css", + input: ` +.grid-container { + display: grid; + grid-template-columns: repeat(<|fim|>); + grid-gap: 10px; +} +`, + llmOutput: "3, 1fr", + expectedCompletion: "3, 1fr", + }, + + { + description: "Should complete PHP function inside a class with comments", + input: `name = $name; + $this->email = $email; + } + + public function <|fim|> + + public function setEmail($email) { + $this->email = $email; + } +}`, + llmOutput: `getDetails() { + return "Name: " . $this->name . ", Email: " . $this->email; + }`, + expectedCompletion: `getDetails() { + return "Name: " . $this->name . ", Email: " . $this->email; + }`, + filename: "User.php", + }, + + { + description: "Should autocomplete PHP function with inline logic", + input: ` +} + +echo calculateArea(5, 3);`, + llmOutput: "return $length * $width;", + expectedCompletion: "return $length * $width;", + filename: "areaCalculator.php", + }, + + { + description: "Should handle PHP completion in the middle of an array", + input: `); + +echo "First color is: " . $colors[0];`, + llmOutput: '"Blue"', + expectedCompletion: '"Blue"', + filename: "colors.php", + }, + { + description: "Should autocomplete React return statements (jsx)", + input: `import React from "react"; + +export const Button = ({ + onClick, + children, +}: { + children: React.ReactNode; + onClick: () => void; +}) => { + return ( +<|fim|> + ); +};`, + llmOutput: ` + {children} + `, + expectedCompletion: ` + {children} + `, + filename: "Button.tsx", + }, + { + description: "Should autocomplete React state initialization", + input: `import React, { useState } from "react"; + +export const Counter = () => { + const [count, setCount] = useState(<|fim|>); + + return ( + + You clicked {count} times + setCount(count + 1)}>Click me + + ); +};`, + llmOutput: "0", + expectedCompletion: "0", + filename: "Counter.tsx", + }, + { + description: "Should autocomplete React component methods", + input: `import React from "react"; + +class Form extends React.Component { + constructor(props) { + super(props); + this.state = { name: '' }; + } + + handleChange = (event) => { + <|fim|> + } + + handleSubmit = (event) => { + event.preventDefault(); + alert('A name was submitted: ' + this.state.name); + } + + render() { + return ( + + + Name: + + + + + ); + } +}`, + llmOutput: "this.setState({ name: event.target.value });", + expectedCompletion: "this.setState({ name: event.target.value });", + filename: "Form.tsx", + }, + { + description: "Should autocomplete Python function definition", + filename: "test.py", + input: `def calculate_area(length, width): + area = length * width + return area + +def calculate_perimeter(length, width): + <|fim|> +`, + llmOutput: `perimeter = 2 * (length + width) + return perimeter`, + expectedCompletion: `perimeter = 2 * (length + width) + return perimeter`, + }, + { + description: "Should complete Python class method with self", + filename: "test.py", + input: `class BankAccount: + def __init__(self, owner, balance=0): + self.owner = owner + self.balance = balance + + def deposit(self, amount): + self.balance += amount + return self.balance + + def withdraw(self, amount): + <|fim|> +`, + llmOutput: `if amount > self.balance: + return "Insufficient funds" + self.balance -= amount + return self.balance`, + expectedCompletion: `if amount > self.balance: + return "Insufficient funds" + self.balance -= amount + return self.balance`, + }, + { + description: "Should autocomplete Python list comprehension", + filename: "test.py", + input: `squares = [x**2 for x in range(10)] +even_squares = [x for x in squares if x % 2 == 0] +print(even_squares) + +odd_squares = [<|fim|> +print(odd_squares) +`, + llmOutput: "x for x in squares if x % 2 != 0", + expectedCompletion: "x for x in squares if x % 2 != 0", + }, + { + description: "Should autocomplete a simple Go function declaration", + filename: "simpleFunction.go", + input: `package main + +import ( + "fmt" +) + +func main() { + fmt.Println("Hello, World!") +} + +func calculateArea<|fim|>`, + llmOutput: `(length float64, width float64) float64 { + return length * width +}`, + expectedCompletion: `(length float64, width float64) float64 { + return length * width +}`, + }, + + { + description: + "Should handle autocomplete in the middle of a Go struct definition", + filename: "structDefinition.go", + input: `package main + +type Person struct { + FirstName string + LastName string + Age int + Address Address +} + +type Address struct { + Street string + City <|fim|> +} + +func main() {}`, + llmOutput: `string + State string + ZipCode string +}`, + expectedCompletion: `string + State string + ZipCode string`, + }, + + { + description: "Should autocomplete a missing Go function body bracket", + filename: "missingBracket.go", + input: `package main + +func add(a int, b int) int { + return a + b<|fim|> + +func multiply(a int, b int) int { + return a * b +} + +func main() { + result1 := add(2, 3) + result2 := multiply(4, 5) + println(result1, result2) +}`, + llmOutput: ` +}`, + expectedCompletion: ` +}`, + }, + { + description: + "Should autocomplete SQL query with nested functions and missing bracket", + filename: "_nested_function.sql", + input: `SELECT name, ROUND(AVG(rating), 2) as avg_rating + FROM movies + WHERE release_year > 2000 AND director = 'Christopher Nolan' + GROUP BY name + HAVING avg_rating > <|fim|>`, + llmOutput: "8.5)", + expectedCompletion: "8.5)", + }, + { + description: + "Should autocomplete SQL script with employee and product tables", + filename: "database.sql", + input: ` + CREATE TABLE employees ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(100) NOT NULL, + age INT NOT NULL, + position VARCHAR(100) + ); + + CREATE TABLE products ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(100), + price DECIMAL(8,2) NOT NULL<|fim|> '0.00', + quantity INT NOT NULL DEFAULT '0' + ); + + INSERT INTO employees (name, age, position) VALUES ('John Doe', 30, 'Developer'); + INSERT INTO products (name, price, quantity) VALUES ('Apple', '1.99', '47'); + + SELECT * FROM products ORDER BY name DESC LIMIT 3; + SELECT * FROM products WHERE price > '0'; + SELECT * FROM products WHERE quantity > '100'; + SELECT * FROM employees WHERE age > 25; + `, + llmOutput: " DEFAULT", + expectedCompletion: " DEFAULT", + }, + { + description: + "Should autocomplete multi-line SQL query with CASE statements", + filename: "case_statement.sql", + input: `SELECT order_id, + order_date, + CASE + WHEN status = 'shipped' THEN 'Completed' + WHEN status = 'pending' THEN 'Pending Approval' + <|fim|> + ELSE 'Unknown' + END as order_status + FROM orders`, + llmOutput: "WHEN status = 'cancelled' THEN 'Cancelled'", + expectedCompletion: "WHEN status = 'cancelled' THEN 'Cancelled'", + }, + { + description: "Should autocomplete HTML paragraph content", + input: ` + + Document + + + <|fim|> + +`, + llmOutput: "This is a paragraph with some sample text.", + expectedCompletion: "This is a paragraph with some sample text.", + filename: "test.html", + }, + { + description: "Should autocomplete HTML attributes within a tag", + input: ` + > + + Title + Description text. + +`, + llmOutput: 'alt="Description of image"', + expectedCompletion: 'alt="Description of image"', + filename: "test.html", + }, + { + description: "Should autocomplete HTML nested tags", + input: ` + Item 1 + Item 2 + Item 3 + `, + llmOutput: ">Item 4", + expectedCompletion: ">Item 4", + filename: "test.html", + }, + + { + description: "Should complete a class method in Ruby", + input: ` +class Greeter + def initialize(name) + @name = name + end + + def greet + puts "Hello, <|fim|> + end +end + +g = Greeter.new("World") +g.greet +`, + llmOutput: "#{@name}!", + expectedCompletion: "#{@name}!", + filename: "greeter.rb", + }, + { + description: "Should complete Ruby if-else block", + input: ` +number = 10 + +if number > 5 + puts "Number is greater than 5" +<|fim|> +end +`, + llmOutput: `else + puts "Number is 5 or less"`, + expectedCompletion: `else + puts "Number is 5 or less"`, + filename: "conditional.rb", + }, + { + description: "Should complete Ruby array method", + input: ` +numbers = [1, 2, 3, 4, 5] +squared_numbers = numbers.<|fim|> +`, + llmOutput: "map { |n| n ** 2 }", + expectedCompletion: "map { |n| n ** 2 }", + filename: "array_methods.rb", + }, + { + description: "Should autocomplete Java within a string", + filename: "App.java", + input: ` +public class App { + public static void main(String[] args) { + String message = "Hello, <|fim|>"; + System.out.println(message); + } +}`, + llmOutput: "World", + expectedCompletion: "World", + }, + { + description: + "Should autocomplete a C++ class definition with a constructor", + filename: "test.cpp", + input: "class Vehicle { public: Vehicle(<|fim|>); };", + llmOutput: "int wheels, double weight", + expectedCompletion: "int wheels, double weight", + }, + { + description: "Should complete a C++ method declaration inside a class", + filename: "test.cpp", + input: + "class Calculator { public: int add(int a, int b); int subtract(int a, int b);<|fim|> };", + llmOutput: " int multiply(int a, int b);", + expectedCompletion: " int multiply(int a, int b);", + }, + { + description: "Should autocomplete C++ for loop syntax", + filename: "test.cpp", + input: ` + int sum = 0; + for (int i = 0; i < 10; <|fim|>) { + sum += i; + } + `, + llmOutput: "i++", + expectedCompletion: "i++", + }, + { + description: "Should autocomplete JSON object inside an array", + filename: "data.json", + input: `{ + "users": [ + { "id": 1, "name": "Alice" }, + { "id": 2, "name": "Bob" } + <|fim|> + ] +}`, + llmOutput: ', { "id": 3, "name": "Charlie" }', + expectedCompletion: ', { "id": 3, "name": "Charlie" }', + }, + { + description: "Should autocomplete within a CSV record", + filename: "test.csv", + input: `Name, Age, City +John Doe, 30, New York +Jane Smith<|fim|>`, + llmOutput: ", 25, Los Angeles", + expectedCompletion: ", 25, Los Angeles", + }, + { + description: + "Should complete CSV record when starting in the middle of a word", + filename: "test.csv", + input: `Product, Price, Quantity +Laptop, 1200, 5 +Smart<|fim|>`, + llmOutput: "phone, 800, 10", + expectedCompletion: "phone, 800, 10", + }, + { + description: "Should complete CSV structure adding closing brackets", + filename: "test.csv", + input: `ID, Name, JoiningDate +1, Alice, 2023-01-10 +2, B<|fim|>`, + llmOutput: "ob, 2023-02-10", + expectedCompletion: "ob, 2023-02-10", + }, + { + description: + "Should autocomplete a Rust function implementation inside a struct", + filename: "main.rs", + input: ` +struct Calculator { + result: f64, +} + +impl Calculator { + fn new() -> Self { + Calculator { result: 0.0 } + } + + fn add(&mut self, number: f64) { + self.result += number; + } + + fn subtract(&mut self, number: f64) { + self.result -= number; + } + + fn multiply(&mut self, number: f64) { + self.result *= number; + } + + fn divide(&mut self, number: f64) { + if number != 0.0 { + self.result /= number; + } else { + println!("Cannot divide by zero."); + } + } + + fn reset(&mut self) { + self.result = 0.0; + } + + fn get_result(&self) -> f64 { + self.result + } + + fn<|fim|> +} +`, + llmOutput: ` divide(&mut self, number: f64) { + if number != 0.0 { + self.result /= number; + } else { + println!("Cannot divide by zero."); + } +}`, + expectedCompletion: ` divide(&mut self, number: f64) { + if number != 0.0 { + self.result /= number; + } else { + println!("Cannot divide by zero."); + }`, + }, + { + description: "Should autocomplete Rust struct definition", + filename: "main.rs", + input: ` +struct User { + id: u32, + username: String, + email: String, + is_active: bool, + <|fim|> +} + +impl User { + fn new(id: u32, username: String, email: String) -> Self { + User { + id, + username, + email, + is_active: true, + } + } +} +`, + llmOutput: `created_at: String, + updated_at: String,`, + expectedCompletion: `created_at: String, + updated_at: String,`, + }, + { + description: "Haskell: Nested pattern matching with let bindings", + filename: "NestedPattern.hs", + input: `module NestedPattern where + +data Tree a = Leaf a | Node (Tree a) (Tree a) + +sumTree :: Num a => Tree a -> a +sumTree (Leaf x) = x +sumTree (Node left right) = + let leftSum = <|fim|> + rightSum = sumTree right + in leftSum + rightSum`, + llmOutput: "sumTree left", + expectedCompletion: "sumTree left", + }, + { + description: "Haskell: Complex function with where clause and guards", + filename: "QuadraticSolver.hs", + input: `module QuadraticSolver where + +solveQuadratic :: (Ord a, Floating a) => a -> a -> a -> Maybe (a, a) +solveQuadratic a b c + | discriminant < 0 = Nothing + | otherwise = Just (x1, x2) + where + discriminant = b^2 - 4*a*c + sqrtD = sqrt discriminant + x1 = (-b + sqrtD) / (2*a) + <|fim|> = (-b - sqrtD) / (2*a)`, + llmOutput: "x2", + expectedCompletion: "x2", + }, + { + description: "Haskell: List comprehension with complex filter", + filename: "PrimeNumbers.hs", + input: `module PrimeNumbers where + +primesUpTo :: Int -> [Int] +primesUpTo n = [x | x <- [2..n], isPrime x] + where isPrime num = <|fim|> && all (\d -> num \`mod\` d /= 0) [2..(floor . sqrt $ fromIntegral num)]`, + llmOutput: "num > 1", + expectedCompletion: "num > 1", + }, + { + description: "Should autocomplete Dart class methods", + filename: "calculator.dart", + input: ` +class Calculator { + double result = 0.0; + + void add(double number) { + result += number; + } + + void multiply(double number) { + result *= number; + } + + <|fim|> + + double getResult() { + return result; + } +}`, + llmOutput: `void subtract(double number) { + result -= number; + }`, + expectedCompletion: `void subtract(double number) { + result -= number; + }`, + }, + { + description: "Should handle string interpolation in Dart", + filename: "greetings.dart", + input: ` +void main() { + var name = "World"; + print('Hello, <|fim|>!'); +}`, + llmOutput: "${name}", + expectedCompletion: "${name}", + }, + { + description: "Should autocomplete within a Dart function body", + filename: "counter.dart", + input: ` +class Counter { + int count = 0; + + void increment() { + count++; + } + + void decrement() { + <|fim|> + + void reset() { + count = 0; + } +}`, + llmOutput: "count--;", + expectedCompletion: "count--;", + }, + { + description: + "Should autocomplete Clojure function definition with missing closing parenthesis", + input: `(defn calculate-sum [a b] + (let [sum (+ a b)] + (println "The sum is" sum) + sum<|fim|>`, + llmOutput: "))", + expectedCompletion: "))", + filename: "test.clj", + }, + { + description: + "Should autocomplete missing part of a Clojure map within a function", + input: `(defn get-user [] + {:username "johndoe" + :email "johndoe@example.com" + :age 30 + <|fim|> + (println "User information loaded"))`, + llmOutput: ':location "Unknown"}', + expectedCompletion: ':location "Unknown"}', + filename: "test.clj", + }, + { + description: + "Should autocomplete inside a Clojure vector within a looping construct", + input: `(defn odd-numbers [] + (loop [nums [1 3 5<|fim|> 9 11]] + (when (seq nums) + (println (first nums)) + (recur (rest nums)))))`, + llmOutput: " 7,", + expectedCompletion: " 7,", + filename: "test.clj", + }, + { + description: "Should autocomplete R function definition", + filename: "calculate.R", + input: ` +calculate_mean <- function(numbers) { + total <- sum(numbers) + <|fim|> +}`, + llmOutput: `mean_value <- total / length(numbers) + return(mean_value)`, + expectedCompletion: `mean_value <- total / length(numbers) + return(mean_value)`, + }, + { + description: "Should complete R loop and print statement", + filename: "loopPrint.R", + input: ` +numbers <- c(1, 2, 3, 4, 5) +for (number in numbers) { + print(<|fim|>) +}`, + llmOutput: "number)", + expectedCompletion: "number)", + }, + { + description: "Should autocomplete R data frame creation", + filename: "dataFrame.R", + input: ` +data <- data.frame( + Name = c("Alice", "Bob", "Charlie"), + Age = c(25, 30, 35), + <|fim|> +)`, + llmOutput: "Height = c(165, 180, 175)", + expectedCompletion: "Height = c(165, 180, 175)", + }, + { + description: "Should autocomplete R if-else statement", + filename: "condition.R", + input: ` +grade <- 85 +if (grade >= 90) { + print("A") +} else if (grade >= 80) { + <|fim|> +} else { + print("C") +}`, + llmOutput: 'print("B")', + expectedCompletion: 'print("B")', + }, + { + description: "Should autocomplete R ggplot2 plot structure", + filename: "plot.R", + input: ` +library(ggplot2) + +ggplot(data=mtcars, aes(x=wt, y=mpg)) + + geom_point() + + <|fim|>`, + llmOutput: "geom_smooth(method='lm', se=FALSE)", + expectedCompletion: "geom_smooth(method='lm', se=FALSE)", + }, + { + description: "Should autocomplete Scala class with a method", + filename: "Person.scala", + input: `class Person(val name: String, val age: Int) { + def greet(): String = { + <|fim|> + } + }`, + llmOutput: 's"Hello, my name is $name and I am $age years old."', + expectedCompletion: 's"Hello, my name is $name and I am $age years old."', + }, + + { + description: "Should handle Scala case class with a missing field", + filename: "Person.scala", + input: `case class Address(city: String, postalCode: String) + case class Person(name: String, age: Int, address: Address) + + val alice = Person("Alice", 30, Address("Wonderland", <|fim|>))`, + llmOutput: '"12345")', + expectedCompletion: '"12345")', + }, + + { + description: "Should autocomplete Scala function with missing body bracket", + filename: "Math.scala", + input: `object MathUtils { + def add(a: Int, b: Int): Int = { + a + b<|fim|> + + def multiply(a: Int, b: Int): Int = { + a * b + } + } + + object Main extends App { + println(MathUtils.add(3, 5)) + println(MathUtils.multiply(4, 6)) + }`, + llmOutput: ` + }`, + expectedCompletion: ` + }`, + }, + { + description: "Should autocomplete C function definition", + filename: "math_utils.c", + input: `#include + +int add(int a, int b) { + return a + b; +} + +int multiply(int a, int b) { + return a * b; +} + +int subtract(int a, int b) { + <|fim|> +} + +int main() { + printf("Result: %d", add(2, 3)); + return 0; +}`, + llmOutput: "return a - b;", + expectedCompletion: "return a - b;", + }, + + { + description: "Should handle C struct with missing field initialization", + filename: "person.c", + input: `#include + +typedef struct { + char name[50]; + int age; + float height; +} Person; + +int main() { + Person alice = {"Alice", 30, <|fim|>}; + printf("Name: %s, Age: %d, Height: %.2f", alice.name, alice.age, alice.height); + return 0; +}`, + llmOutput: "5.5", + expectedCompletion: "5.5", + }, + + { + description: "Should autocomplete C function with missing body bracket", + filename: "area.c", + input: `#include + +double calculateCircleArea(double radius) { + const double pi = 3.14159; + return pi * radius * radius;<|fim|> + +double calculateRectangleArea(double length, double width) { + return length * width; +} + +int main() { + printf("Circle Area: %.2f", calculateCircleArea(5.0)); + printf("Rectangle Area: %.2f", calculateRectangleArea(4.0, 6.0)); + return 0; +}`, + llmOutput: ` +}`, + expectedCompletion: ` +}`, + }, + { + description: "Should autocomplete a simple Kotlin function declaration", + filename: "simpleFunction.kt", + input: ` +fun main() { + println("Hello, World!") +} + +fun calculateArea(length: Double, width: Double): Double <|fim|>`, + llmOutput: `{ + return length * width +}`, + expectedCompletion: `{ + return length * width +}`, + }, + { + description: "Should handle autocomplete inside a Kotlin data class", + filename: "dataClass.kt", + input: ` +data class User( + val id: Int, + val name: String, + val email: String, + <|fim|> +)`, + llmOutput: "val age: Int", + expectedCompletion: "val age: Int", + }, + { + description: + "Should complete Kotlin if-else structure with missing brackets", + filename: "controlStructure.kt", + input: ` +fun getMax(a: Int, b: Int): Int { + if (a > b<|fim|> + } else { + return b + } +}`, + llmOutput: `) { + return a`, + expectedCompletion: `) { + return a`, + }, + { + description: "Should autocomplete Solidity function definition", + filename: "SimpleStorage.sol", + input: ` +pragma solidity ^0.8.0; + +contract SimpleStorage { + uint private data; + + function set(uint x) public { + data = x; + } + + function get() public view returns (uint) { + <|fim|> + } +} + `, + llmOutput: "return data;", + expectedCompletion: "return data;", + }, + + { + description: "Should autocomplete Solidity event with parameters", + filename: "EventExample.sol", + input: ` +pragma solidity ^0.8.0; + +contract EventExample { + event DataStored(uint indexed id, string content); + + function storeData(uint id, string memory content) public { + emit DataStored(<|fim|>); + } +} + `, + llmOutput: "id, content", + expectedCompletion: "id, content", + }, + + { + description: "Should handle Solidity struct definition completion", + filename: "StructDefinition.sol", + input: ` +pragma solidity ^0.8.0; + +contract StructExample { + struct Person { + string name; + uint age; + address wallet; + } + + Person[] private people; + + function addPerson(string memory name, uint age, address wallet) public { + people.push(Person(name, age, wallet)); + } + + function getFirstPerson<|fim|> +} + `, + llmOutput: `() public view returns (string memory, uint, address) { + if (people.length > 0) { + Person storage person = people[0]; + return (person.name, person.age, person.wallet); + } + return ("", 0, address(0)); + }`, + expectedCompletion: `() public view returns (string memory, uint, address) { + if (people.length > 0) { + Person storage person = people[0]; + return (person.name, person.age, person.wallet); + } + return ("", 0, address(0)); + }`, + }, + { + description: "Should autocomplete TypeScript interface properties", + filename: "User.ts", + input: ` +interface User { + id: number; + name: string; + e<|fim|> +} +`, + llmOutput: `mail: string; + age: number; +}`, + expectedCompletion: `mail: string; + age: number;`, + }, + { + description: "Should autocomplete TypeScript interface declarations", + filename: "autocomplete.ts", + input: `interface AutocompleteDiffSnippet extends BaseAutocompleteSnippet {} + +interface AutocompleteCodeSnippet`, + llmOutput: ` extends BaseAutocompleteSnippet { + filepath: string; +}`, + expectedCompletion: ` extends BaseAutocompleteSnippet { + filepath: string; +}`, + }, + { + description: + "Should autocomplete a TypeScript arrow function inside a variable assignment", + filename: "mathOperations.ts", + input: ` +const addNumbers = (a: number, b: number): number => { + return a + b; +}; + +const multiplyNumbers = (a: number, b: number): number => { + re<|fim|> +} + +console.log(multiplyNumbers(2, 3)); +`, + llmOutput: "turn a * b;", + expectedCompletion: "turn a * b;", + }, + + // TODO + // { + // description: + // "Should handle autocomplete inside a nested TypeScript class method", + // filename: "Account.ts", + // input: ` + // class Account { + // private balance: number = 0; + + // deposit(amount: number) { + // this.balance += amount; + // return this.balance; + // } + + // withdraw(amount: number) { + // if (amount > this.balance) { + // throw new Error("Insufficient funds"); + // } + // this.balance -= amount; + // return thi<|fim|> + // } + // } + // `, + // llmOutput: `s.balance;`, + // expectedCompletion: `s.balance;`, + // }, + + // TODO + // { + // description: "Should autocomplete a TypeScript generic function", + // filename: "GenericFunction.ts", + // input: ` + // function identity(arg: T): T { + // return ar<|fim|> + // } + + // console.log(identity(5)); + // `, + // llmOutput: `g;`, + // expectedCompletion: `g;`, + // }, + + // TODO + // { + // description: + // "Should autocomplete a TypeScript promise within an asynchronous function", + // filename: "asyncFunction.ts", + // input: ` + // async function fetchData(url: string): Promise { + // const response = await fetch(url); + // <|fim|> + // return data; + // } + + // fetchData('https://api.example.com/data'); + // `, + // llmOutput: `const data = await response.json();`, + // expectedCompletion: `const data = await response.json();`, + // }, + + { + description: + "Should autocomplete a C# class with a constructor and property", + filename: "Person.cs", + input: `using System; + +public class Person +{ + public string Name { get; set; } + public int Age { get; set; } + + public Person(string name, int <|fim|> +}`, + llmOutput: `age) + { + Name = name; + Age = age; + }`, + expectedCompletion: `age) + { + Name = name; + Age = age; + }`, + }, + { + description: "Should autocomplete a C# interface method", + filename: "IGreetable.cs", + input: `public interface IGreetable +{ + void <|fim|> +}`, + llmOutput: "Greet();", + expectedCompletion: "Greet();", + }, + { + description: "Should autocomplete inside C# method with if condition", + filename: "Calculator.cs", + input: `using System; + +public class Calculator +{ + public int Add(int a, int b) + { + if(<|fim|>) + { + return a + b; + } + return 0; + } +}`, + llmOutput: "a > 0 && b > 0", + expectedCompletion: "a > 0 && b > 0", + }, + { + description: "Should complete a simple Julia function", + filename: "simpleFunction.jl", + input: `function calculate_area(length, width) + <|fim|> +end +`, + llmOutput: "return length * width", + expectedCompletion: "return length * width", + }, + { + description: "Should autocomplete Julia for loop", + filename: "loop.jl", + input: `numbers = [1, 2, 3, 4, 5] +squared_numbers = [] + +for num in numbers + <|fim|> +end + +println(squared_numbers) +`, + llmOutput: "push!(squared_numbers, num^2)", + expectedCompletion: "push!(squared_numbers, num^2)", + }, + { + description: "Should complete a Julia struct definition", + filename: "structDefinition.jl", + input: `struct Person + first_name::String + last_name::String + age::Int + address::Address +end + +struct Address + street::String + city::String + <|fim|> +end +`, + llmOutput: `state::String + zip_code::String`, + expectedCompletion: `state::String + zip_code::String`, + }, + { + description: "Should complete a Julia dictionary access", + filename: "dictionary.jl", + input: `grades = Dict("Alice" => 90, "Bob" => 85, "Eve" => 88) + +function get_grade(student_name) + return grades[<|fim|>] +end + +println(get_grade("Alice")) # Should print 90 +`, + llmOutput: "student_name", + expectedCompletion: "student_name", + }, + { + description: "Should complete a Julia module declaration", + filename: "moduleDeclaration.jl", + input: `module MathOperations + +export add, subtract + +function add(a, b) + return a + b +end + +function subtract(a, b) + return a - b +end + +<|fim|> +`, + llmOutput: "end", + expectedCompletion: "end", + }, + { + description: "Should complete F# let-binding with function definition", + filename: "mathModule.fs", + input: `module MathModule + +let calculateArea length width = + <|fim|>`, + llmOutput: "length * width", + expectedCompletion: "length * width", + }, + + { + description: "Should complete incomplete F# type definition", + filename: "personType.fs", + input: `type Person = { + FirstName: string + LastName: string + Age: int<|fim|> +} + +let john = { FirstName = "John"; LastName = "Doe"; Age = 30 }`, + llmOutput: ` + Address: string +}`, + expectedCompletion: ` + Address: string`, + }, + + { + description: "Should complete F# pattern matching expression", + filename: "patternMatching.fs", + input: `let describeNumber number = + match number with + | 0 -> "Zero" + | 1 -> "One" + | <|fim|>`, + llmOutput: `2 -> "Two" + | _ -> "Other"`, + expectedCompletion: `2 -> "Two" + | _ -> "Other"`, + }, + + { + description: "Should complete F# list comprehension expression", + filename: "listComprehension.fs", + input: `let squares = [ for x in 1..10 -> x * x ] +let evenSquares = [ for x in squares do if x % 2 = 0 then yield x ] +let oddSquares = [<|fim|>]`, + llmOutput: " for x in squares do if x % 2 <> 0 then yield x ]", + expectedCompletion: " for x in squares do if x % 2 <> 0 then yield x ]", + }, + { + description: "Should complete an F# recursive function", + filename: "recursiveFunctions.fs", + input: `let rec factorial n = + if n <= 1 then 1 + else n <|fim|> factorial (n - 1)`, + llmOutput: "*", + expectedCompletion: "*", + }, + { + description: "Should complete F# member method inside a class type", + filename: "bankAccount.fs", + input: `type BankAccount(owner: string, initialBalance: float) = + let mutable balance = initialBalance + + member this.Deposit amount = + balance <- balance + amount + this + <|fim|>`, + llmOutput: ` + member this.Withdraw amount = + if amount > balance then + failwith "Insufficient funds" + balance <- balance - amount + this`, + expectedCompletion: ` + member this.Withdraw amount = + if amount > balance then + failwith "Insufficient funds" + balance <- balance - amount + this`, + }, + + { + description: "Should complete F# async workflow function", + filename: "asyncWorkflow.fs", + input: `let fetchDataAsync url = + async { + use client = new System.Net.Http.HttpClient() + <|fim|> + }`, + llmOutput: `let! response = client.GetStringAsync(url) + return response`, + expectedCompletion: `let! response = client.GetStringAsync(url) + return response`, + }, + { + description: "Should autocomplete SCSS nested class starting inside a rule", + filename: "styles.scss", + input: `nav { + display: flex; + justify-content: space-between; + .logo { + font-size: 1.5rem; + color: #333; + + .brand<|fim|> + } + + ul { + list-style: none; + display: flex; + gap: 1rem; + } +}`, + llmOutput: `-name { + font-weight: bold; + text-transform: uppercase; +}`, + expectedCompletion: `-name { + font-weight: bold; + text-transform: uppercase; +}`, + }, + + { + description: "Should handle SCSS mixin within a class", + filename: "styles.scss", + input: `.card { + border: 1px solid #ccc; + padding: 10px; + + @include<|fim|> + + &:hover { + box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1); + } +}`, + llmOutput: " transition(all 0.3s ease);", + expectedCompletion: " transition(all 0.3s ease);", + }, + + { + description: + "Should autocomplete SCSS variable in the middle of a statement", + filename: "styles.scss", + input: `$primary-color: #007bff; +$secondary-color: #6c757d; + +.button { + background-color: <|fim|> color; + padding: 10px 15px; + border: none; + color: #fff; + border-radius: 4px; +}`, + llmOutput: "$primary-", + expectedCompletion: "$primary-", + }, + { + description: "Should autocomplete Vue component method", + filename: "MyComponent.vue", + input: ` + + Increment + Count: {{ count }} + + + + +`, + llmOutput: `() { + this.count = 0; + }`, + expectedCompletion: `() { + this.count = 0; + }`, + }, + { + description: "Should autocomplete Vue computed property", + filename: "UserComponent.vue", + input: ` + + User Full Name: {{ fullName }} + + + + +`, + llmOutput: `() { + return this.firstName + ' ' + this.lastName; + }`, + expectedCompletion: `() { + return this.firstName + ' ' + this.lastName; + }`, + }, + { + description: "Should autocomplete Vue method using props", + filename: "TodoItem.vue", + input: ` + + {{ title }} + Complete + + + + +`, + llmOutput: "this.completed", + expectedCompletion: "this.completed", + }, + { + description: "Should autocomplete Svelte reactive statement", + filename: "Counter.svelte", + input: ` + + + + Clicked {count} times + +`, + llmOutput: "doubledCount = count * 2", + expectedCompletion: "doubledCount = count * 2", + }, + + { + description: "Should autocomplete Svelte component inside HTML", + filename: "NestedComponent.svelte", + input: ` + + + + Hello Svelte + /> + +`, + llmOutput: 'name="World"', + expectedCompletion: 'name="World"', + }, + + { + description: "Should handle autocomplete in Svelte each block", + filename: "List.svelte", + input: ` + + + + {#each items as item} + {item} + {/each<|fim|> + +`, + llmOutput: "}", + expectedCompletion: "}", + }, + { + description: + "Should handle autocomplete in two similar TypeScript functions", + filename: "List.svelte", + input: ` +import { createClient, RedisClientType } from "redis"; +import { IKeyValueStore } from "./index.js"; + +export class RedisKeyValueStore implements IKeyValueStore { + private client: RedisClientType; + + constructor(redisUrl: string) { + this.client = createClient({ + url: redisUrl + .replace("https://", "redis://") + .replace("http://", "redis://"), + }); + this.client.on("connect", () => console.log("Redis Connected")); + this.client.on("error", (err) => console.log("Redis Client Error", err)); + this.client.connect(); + } + public async has(tableName: string, key: string): Promise { + return (await this.client.exists(this._getKey(tableName, key))) > 0; + } + + public async keys(tableName: string): Promise { + const keys = await this.client.keys(this._getTableKey(tableName)); + return keys.map((key) => key.split("::")[1]); + } + + public async put( + tableName: string, + key: string, + value: string, + ): Promise { + await this.client.set(this._getKey(tableName, key), value); + } + + public async get( + tableName: string, + key: string, + ): Promise { + const value = await this.client.get(this._getKey(tableName, key)); + return value ?? undefined; + } + + public async deleteAll(tableName: string): Promise { + await this.client.del(this._getTableKey(tableName)); + } + +<|fim|> + + + public async remove(tableName: string, key: string): Promise { + const result = await this.client.del(this._getKey(tableName, key)); + return result > 0; + } +} +`, + llmOutput: ` public async delete(tableName: string, key: string): Promise { + await this.client.del(this._getKey(tableName, key)); + }`, + expectedCompletion: ` public async delete(tableName: string, key: string): Promise { + await this.client.del(this._getKey(tableName, key)); + }`, + }, +]; diff --git a/core/autocomplete/filtering/test/util.ts b/core/autocomplete/filtering/test/util.ts new file mode 100644 index 0000000000..db8d991317 --- /dev/null +++ b/core/autocomplete/filtering/test/util.ts @@ -0,0 +1,75 @@ +import fs from "node:fs"; +import path from "node:path"; + +import MockLLM from "../../../llm/llms/Mock"; +import { testConfigHandler, testIde } from "../../../test/fixtures"; +import { CompletionProvider } from "../../CompletionProvider"; +import { AutocompleteInput } from "../../util/types"; + +const FIM_DELIMITER = "<|fim|>"; + +function parseFimExample(text: string): { prefix: string; suffix: string } { + const [prefix, suffix] = text.split(FIM_DELIMITER); + return { prefix, suffix }; +} + +export interface AutocompleteFileringTestInput { + description: string; + filename: string; + input: string; + llmOutput: string; + expectedCompletion: string | null | undefined; + options?: { + only?: boolean; + }; +} + +export async function testAutocompleteFiltering( + test: AutocompleteFileringTestInput, +) { + const { prefix, suffix } = parseFimExample(test.input); + + // Setup necessary objects + const llm = new MockLLM({ + model: "mock", + }); + llm.completion = test.llmOutput; + const ide = testIde; + const configHandler = testConfigHandler; + + // Create a real file + const [workspaceDir] = await ide.getWorkspaceDirs(); + const filepath = path.join(workspaceDir, test.filename); + fs.writeFileSync(filepath, test.input.replace(FIM_DELIMITER, "")); + + // Prepare completion input and provider + const completionProvider = new CompletionProvider( + configHandler, + ide, + async () => llm, + () => {}, + async () => [], + ); + + const line = prefix.split("\n").length - 1; + const character = prefix.split("\n")[line].length; + const autocompleteInput: AutocompleteInput = { + completionId: "test-completion-id", + filepath, + pos: { + line, + character, + }, + recentlyEditedFiles: [], + recentlyEditedRanges: [], + }; + + // Generate a completion + const result = await completionProvider.provideInlineCompletionItems( + autocompleteInput, + undefined, + ); + + // Ensure that we return the text that is wanted to be displayed + expect(result?.completion).toEqual(test.expectedCompletion); +} diff --git a/core/autocomplete/generation/CompletionStreamer.ts b/core/autocomplete/generation/CompletionStreamer.ts new file mode 100644 index 0000000000..d44957d15b --- /dev/null +++ b/core/autocomplete/generation/CompletionStreamer.ts @@ -0,0 +1,69 @@ +import { CompletionOptions, ILLM } from "../.."; +import { StreamTransformPipeline } from "../filtering/streamTransforms/StreamTransformPipeline"; +import { HelperVars } from "../util/HelperVars"; + +import { GeneratorReuseManager } from "./GeneratorReuseManager"; + +export class CompletionStreamer { + private streamTransformPipeline = new StreamTransformPipeline(); + private generatorReuseManager: GeneratorReuseManager; + + constructor(onError: (err: any) => void) { + this.generatorReuseManager = new GeneratorReuseManager(onError); + } + + async *streamCompletionWithFilters( + token: AbortSignal, + llm: ILLM, + prefix: string, + suffix: string, + prompt: string, + multiline: boolean, + completionOptions: Partial | undefined, + helper: HelperVars, + ) { + // Try to reuse pending requests if what the user typed matches start of completion + const generator = this.generatorReuseManager.getGenerator( + prefix, + (abortSignal: AbortSignal) => + llm.supportsFim() + ? llm.streamFim(prefix, suffix, abortSignal, completionOptions) + : llm.streamComplete(prompt, abortSignal, { + ...completionOptions, + raw: true, + }), + multiline, + ); + + // Full stop means to stop the LLM's generation, instead of just truncating the displayed completion + const fullStop = () => + this.generatorReuseManager.currentGenerator?.cancel(); + + // LLM + const generatorWithCancellation = async function* () { + for await (const update of generator) { + if (token.aborted) { + return; + } + yield update; + } + }; + + const initialGenerator = generatorWithCancellation(); + const transformedGenerator = helper.options.transform + ? this.streamTransformPipeline.transform( + initialGenerator, + prefix, + suffix, + multiline, + completionOptions?.stop || [], + fullStop, + helper, + ) + : initialGenerator; + + for await (const update of transformedGenerator) { + yield update; + } + } +} diff --git a/core/autocomplete/generation/GeneratorReuseManager.test.ts b/core/autocomplete/generation/GeneratorReuseManager.test.ts new file mode 100644 index 0000000000..0390599032 --- /dev/null +++ b/core/autocomplete/generation/GeneratorReuseManager.test.ts @@ -0,0 +1,215 @@ +import { jest } from "@jest/globals"; +import { GeneratorReuseManager } from "./GeneratorReuseManager"; + +function createMockGenerator( + data: string[], + delay: number = 0, +): (abortSignal: AbortSignal) => AsyncGenerator { + const mockGenerator = async function* () { + for (const chunk of data) { + yield chunk; + + if (delay > 0) { + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } + }; + const newGenerator = jest + .fn<() => AsyncGenerator>() + .mockReturnValue(mockGenerator()); + + return newGenerator; +} + +describe("GeneratorReuseManager", () => { + let reuseManager: GeneratorReuseManager; + let onErrorMock: jest.Mock; + + beforeEach(() => { + onErrorMock = jest.fn(); + reuseManager = new GeneratorReuseManager(onErrorMock); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test("creates new generator when there is no current generator", async () => { + const data = ["hello ", "world"]; + const newGenerator = createMockGenerator(data); + + const prefix = ""; + + const generator = reuseManager.getGenerator(prefix, newGenerator, true); + + const output: string[] = []; + for await (const chunk of generator) { + output.push(chunk); + } + + expect(output).toEqual(data); + expect(newGenerator).toHaveBeenCalledTimes(1); + }); + + test("reuses generator when prefix matches pending completion", async () => { + const newGenerator = createMockGenerator(["llo ", "world"]); + + // First call with initial prefix + const prefix1 = "he"; + const generator1 = reuseManager.getGenerator(prefix1, newGenerator, true); + + const output1: string[] = []; + for await (const chunk of generator1) { + output1.push(chunk); + } + + expect(output1).toEqual(["llo ", "world"]); + + // Second call with extended prefix that matches pending completion + const prefix2 = "hello "; + const generator2 = reuseManager.getGenerator(prefix2, newGenerator, true); + + const output2: string[] = []; + for await (const chunk of generator2) { + output2.push(chunk); + } + + expect(output2).toEqual(["world"]); + + // Ensure generator was reused (newGenerator should be called only once) + expect(newGenerator).toHaveBeenCalledTimes(1); + }); + + test("creates new generator when prefix does not match pending completion", async () => { + const data = ["goodbye ", "world"]; + const newGenerator = createMockGenerator(data); + + // Initial generator with different prefix + reuseManager.pendingGeneratorPrefix = "hello "; + reuseManager.pendingCompletion = "world"; + + const prefix = "good"; + const generator = reuseManager.getGenerator(prefix, newGenerator, true); + + const output: string[] = []; + for await (const chunk of generator) { + output.push(chunk); + } + + expect(output).toEqual(data); + // Ensure a new generator was created + expect(newGenerator).toHaveBeenCalledTimes(1); + }); + + test("handles multiline=false by stopping at newline", async () => { + const data = ["first line\n", "second line"]; + const newGenerator = createMockGenerator(data); + + const prefix = ""; + const generator = reuseManager.getGenerator(prefix, newGenerator, false); + + const output: string[] = []; + for await (const chunk of generator) { + output.push(chunk); + } + + expect(output).toEqual(["first line"]); + // Ensure it stops after the first newline + }); + + test("handles multiline=true by not stopping at newline", async () => { + const data = ["first line\n", "second line"]; + const newGenerator = createMockGenerator(data); + + const prefix = ""; + const generator = reuseManager.getGenerator(prefix, newGenerator, true); + + const output: string[] = []; + for await (const chunk of generator) { + output.push(chunk); + } + + expect(output).toEqual(data); + }); + + test("cancels previous generator when creating a new one", async () => { + const data1 = ["data from generator 1", "not generated"]; + const data2 = ["data from generator 2"]; + + const newGenerator1 = createMockGenerator(data1, 1000); // Delay so we have the chance to cancel it + const newGenerator2 = createMockGenerator(data2); + + const prefix1 = "prefix1"; + const prefix2 = "prefix2"; + + // First generator + const generator1 = reuseManager.getGenerator(prefix1, newGenerator1, true); + const output1: string[] = []; + for await (const chunk of generator1) { + output1.push(chunk); + // Simulate the generator being canceled before completing + reuseManager.currentGenerator?.cancel(); + } + + expect(output1.length).toEqual(1); + expect(output1[0]).toEqual(data1[0]); + + // Second generator + const generator2 = reuseManager.getGenerator(prefix2, newGenerator2, true); + const output2: string[] = []; + for await (const chunk of generator2) { + output2.push(chunk); + } + + expect(output2).toEqual(data2); + }); + + test("calls onError when generator throws an error", async () => { + const error = new Error("Generator error"); + const mockGenerator = async function* () { + throw error; + }; + const newGenerator = jest + .fn<() => AsyncGenerator>() + .mockReturnValue(mockGenerator()); + + const prefix = ""; + const generator = reuseManager.getGenerator(prefix, newGenerator, true); + + const output: string[] = []; + await expect(async () => { + for await (const chunk of generator) { + output.push(chunk); + } + }).not.toThrow(); // getGenerator handles errors internally + + expect(onErrorMock).toHaveBeenCalledWith(error); + expect(output).toEqual([]); + }); + + test("handles backspacing by creating new generator when prefix is shorter", async () => { + const data = ["hello world"]; + const newGenerator1 = createMockGenerator(data); + const newGenerator2 = createMockGenerator(data); + + // First prefix + const prefix1 = "hello world"; + const generator1 = reuseManager.getGenerator(prefix1, newGenerator1, true); + const output1: string[] = []; + for await (const chunk of generator1) { + output1.push(chunk); + } + + // Simulate backspace (prefix is shorter) + const prefix2 = "hello worl"; + const generator2 = reuseManager.getGenerator(prefix2, newGenerator2, true); + const output2: string[] = []; + for await (const chunk of generator2) { + output2.push(chunk); + } + + // Ensure a new generator was created + expect(newGenerator1).toHaveBeenCalledTimes(1); + expect(newGenerator2).toHaveBeenCalledTimes(1); + }); +}); diff --git a/core/autocomplete/generation/GeneratorReuseManager.ts b/core/autocomplete/generation/GeneratorReuseManager.ts new file mode 100644 index 0000000000..e099b92f46 --- /dev/null +++ b/core/autocomplete/generation/GeneratorReuseManager.ts @@ -0,0 +1,77 @@ +import { ListenableGenerator } from "./ListenableGenerator"; + +export class GeneratorReuseManager { + currentGenerator: ListenableGenerator | undefined; + pendingGeneratorPrefix: string | undefined; + pendingCompletion = ""; + + constructor(private readonly onError: (err: any) => void) {} + + private _createListenableGenerator( + abortController: AbortController, + gen: AsyncGenerator, + prefix: string, + ) { + this.currentGenerator?.cancel(); + + const listenableGen = new ListenableGenerator(gen, this.onError, abortController); + listenableGen.listen((chunk) => (this.pendingCompletion += chunk ?? "")); + + this.pendingGeneratorPrefix = prefix; + this.pendingCompletion = ""; + this.currentGenerator = listenableGen; + } + + private shouldReuseExistingGenerator(prefix: string): boolean { + return ( + !!this.currentGenerator && + !!this.pendingGeneratorPrefix && + (this.pendingGeneratorPrefix + this.pendingCompletion).startsWith( + prefix, + ) && + // for e.g. backspace + this.pendingGeneratorPrefix?.length <= prefix?.length + ); + } + + async *getGenerator( + prefix: string, + newGenerator: (abortSignal: AbortSignal) => AsyncGenerator, + multiline: boolean, + ): AsyncGenerator { + // If we can't reuse, then create a new generator + if (!this.shouldReuseExistingGenerator(prefix)) { + // Create a wrapper over the current generator to fix the prompt + const abortController = new AbortController(); + this._createListenableGenerator(abortController, newGenerator(abortController.signal), prefix); + } + + // Already typed characters are those that are new in the prefix from the old generator + let typedSinceLastGenerator = + prefix.slice(this.pendingGeneratorPrefix?.length) || ""; + for await (let chunk of this.currentGenerator?.tee() ?? []) { + if (!chunk) { + continue; + } + + // Ignore already typed characters in the completion + while (chunk.length && typedSinceLastGenerator.length) { + if (chunk[0] === typedSinceLastGenerator[0]) { + typedSinceLastGenerator = typedSinceLastGenerator.slice(1); + chunk = chunk.slice(1); + } else { + break; + } + } + + // Break at newline unless we are in multiline mode + const newLineIndex = chunk.indexOf("\n"); + if (newLineIndex >= 0 && !multiline) { + yield chunk.slice(0, newLineIndex); + break; + } else if (chunk !== "") { + yield chunk; + } + } + } +} diff --git a/core/autocomplete/generation/ListenableGenerator.test.ts b/core/autocomplete/generation/ListenableGenerator.test.ts new file mode 100644 index 0000000000..916d8dbef3 --- /dev/null +++ b/core/autocomplete/generation/ListenableGenerator.test.ts @@ -0,0 +1,174 @@ +import { jest } from "@jest/globals"; + +import { ListenableGenerator } from "./ListenableGenerator"; + +describe("ListenableGenerator", () => { + // Helper function to create an async generator + async function* asyncGenerator(values: T[], delay = 0) { + for (const value of values) { + if (delay > 0) { + await new Promise((resolve) => setTimeout(resolve, delay)); + } + yield value; + } + } + + it("should yield values from the source generator via tee()", async () => { + const values = [1, 2, 3]; + const source = asyncGenerator(values); + const onError = jest.fn(); + + const lg = new ListenableGenerator( + source, + onError, + new AbortController(), + ); + + const result: number[] = []; + for await (const value of lg.tee()) { + result.push(value); + } + + expect(result).toEqual(values); + expect(onError).not.toHaveBeenCalled(); + }); + + it("should allow listeners to receive values", async () => { + const values = [1, 2, 3]; + const source = asyncGenerator(values, 10); // Introduce delay to simulate async behavior + const onError = jest.fn(); + + const lg = new ListenableGenerator( + source, + onError, + new AbortController(), + ); + + const listener = jest.fn(); + + // Add listener after some delay to simulate late subscription + setTimeout(() => { + lg.listen(listener); + }, 15); + + // Wait for generator to finish + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(listener).toHaveBeenCalledWith(1); + expect(listener).toHaveBeenCalledWith(2); + expect(listener).toHaveBeenCalledWith(3); + // Listener should receive null at the end + expect(listener).toHaveBeenCalledWith(null); + }); + + it("should buffer values for listeners added after some values have been yielded", async () => { + const values = [1, 2, 3]; + const source = asyncGenerator(values, 10); + const onError = jest.fn(); + + const lg = new ListenableGenerator( + source, + onError, + new AbortController(), + ); + + const initialListener = jest.fn(); + + lg.listen(initialListener); + + // Wait for the first value to be yielded + await new Promise((resolve) => setTimeout(resolve, 15)); + + // Add a second listener + const newListener = jest.fn(); + lg.listen(newListener); + + // Wait for generator to finish + await new Promise((resolve) => setTimeout(resolve, 50)); + + // Both listeners should have received all values + [initialListener, newListener].forEach((listener) => { + expect(listener).toHaveBeenCalledWith(1); + expect(listener).toHaveBeenCalledWith(2); + expect(listener).toHaveBeenCalledWith(3); + expect(listener).toHaveBeenCalledWith(null); + }); + }); + + it("should handle cancellation", async () => { + const values = [1, 2, 3, 4, 5]; + const source = asyncGenerator(values, 10); + const onError = jest.fn(); + + const lg = new ListenableGenerator( + source, + onError, + new AbortController(), + ); + + const result: number[] = []; + const teeIterator = lg.tee(); + + const consume = async () => { + for await (const value of teeIterator) { + result.push(value); + if (value === 3) { + lg.cancel(); + } + } + }; + + await consume(); + + expect(result).toEqual([1, 2, 3]); + expect(lg["_isEnded"]).toBe(true); + }); + + it("should call onError when the source generator throws an error", async () => { + async function* errorGenerator() { + yield 1; + throw new Error("Test error"); + } + + const source = errorGenerator(); + const onError = jest.fn(); + + const lg = new ListenableGenerator( + source, + onError, + new AbortController(), + ); + + const result: number[] = []; + for await (const value of lg.tee()) { + result.push(value); + } + + expect(result).toEqual([1]); + expect(onError).toHaveBeenCalledTimes(1); + expect(onError).toHaveBeenCalledWith(new Error("Test error")); + }); + + it("should notify listeners when the generator ends", async () => { + const values = [1, 2, 3]; + const source = asyncGenerator(values); + const onError = jest.fn(); + + const lg = new ListenableGenerator( + source, + onError, + new AbortController(), + ); + + const listener = jest.fn(); + lg.listen(listener); + + // Wait for the generator to finish + await new Promise((resolve) => setTimeout(resolve, 10)); + + expect(listener).toHaveBeenCalledWith(1); + expect(listener).toHaveBeenCalledWith(2); + expect(listener).toHaveBeenCalledWith(3); + expect(listener).toHaveBeenCalledWith(null); + }); +}); diff --git a/core/autocomplete/generation/ListenableGenerator.ts b/core/autocomplete/generation/ListenableGenerator.ts new file mode 100644 index 0000000000..e6b36019b4 --- /dev/null +++ b/core/autocomplete/generation/ListenableGenerator.ts @@ -0,0 +1,79 @@ +export class ListenableGenerator { + private _source: AsyncGenerator; + private _buffer: T[] = []; + private _listeners: Set<(value: T) => void> = new Set(); + private _isEnded = false; + private _abortController: AbortController + + constructor( + source: AsyncGenerator, + private readonly onError: (e: any) => void, + abortController: AbortController + ) { + this._source = source; + this._abortController = abortController; + this._start(); + } + + public cancel() { + this._abortController.abort(); + this._isEnded = true; + } + + private async _start() { + try { + for await (const value of this._source) { + if (this._isEnded) { + break; + } + this._buffer.push(value); + for (const listener of this._listeners) { + listener(value); + } + } + } catch (e) { + this.onError(e); + } finally { + this._isEnded = true; + for (const listener of this._listeners) { + listener(null as any); + } + } + } + + listen(listener: (value: T) => void) { + this._listeners.add(listener); + for (const value of this._buffer) { + listener(value); + } + if (this._isEnded) { + listener(null as any); + } + } + + async *tee(): AsyncGenerator { + try { + let i = 0; + while (i < this._buffer.length) { + yield this._buffer[i++]; + } + while (!this._isEnded) { + let resolve: (value: any) => void; + const promise = new Promise((res) => { + resolve = res; + this._listeners.add(resolve!); + }); + await promise; + this._listeners.delete(resolve!); + + // Possible timing caused something to slip in between + // timers so we iterate over the buffer + while (i < this._buffer.length) { + yield this._buffer[i++]; + } + } + } finally { + // this._listeners.delete(resolve!); + } + } +} diff --git a/core/autocomplete/postprocessing.ts b/core/autocomplete/postprocessing/index.ts similarity index 57% rename from core/autocomplete/postprocessing.ts rename to core/autocomplete/postprocessing/index.ts index cb3ada4594..ed7c41f05c 100644 --- a/core/autocomplete/postprocessing.ts +++ b/core/autocomplete/postprocessing/index.ts @@ -1,6 +1,7 @@ -import type { ILLM } from "../index.js"; -import { longestCommonSubsequence } from "../util/lcs.js"; -import { lineIsRepeated } from "./streamTransforms/lineStream.js"; +import { longestCommonSubsequence } from "../../util/lcs.js"; +import { lineIsRepeated } from "../filtering/streamTransforms/lineStream.js"; + +import type { ILLM } from "../../index.js"; function rewritesLineAbove(completion: string, prefix: string): boolean { const lineAbove = prefix @@ -42,6 +43,14 @@ function isExtremeRepetition(completion: string): boolean { } return false; } +function isOnlyWhitespace(completion: string): boolean { + const whitespaceRegex = /^[\s]+$/; + return whitespaceRegex.test(completion); +} + +function isBlank(completion: string): boolean { + return completion.trim().length === 0; +} export function postprocessCompletion({ completion, @@ -55,7 +64,12 @@ export function postprocessCompletion({ suffix: string; }): string | undefined { // Don't return empty - if (completion.trim().length <= 0) { + if (isBlank(completion)) { + return undefined; + } + + // Don't return whitespace + if (isOnlyWhitespace(completion)) { return undefined; } @@ -69,9 +83,6 @@ export function postprocessCompletion({ return undefined; } - // Remove trailing whitespace - completion = completion.trimEnd(); - if (llm.model.includes("codestral")) { // Codestral sometimes starts with an extra space if (completion[0] === " " && completion[1] !== " ") { @@ -81,24 +92,37 @@ export function postprocessCompletion({ } } - // If completion starts with multiple whitespaces, but the cursor is at the end of the line - // then it should probably be on a new line - if ( - (completion.startsWith(" ") || completion.startsWith("\t")) && - !prefix.endsWith("\n") && - (suffix.startsWith("\n") || suffix.trim().length === 0) - ) { - // completion = "\n" + completion; - return undefined; + if (llm.model.includes("granite")) { + // Granite tends to repeat the start of the line in the completion output + let prefixEnd = prefix.split("\n").pop(); + if (prefixEnd) { + if (completion.startsWith(prefixEnd)) { + completion = completion.slice(prefixEnd.length); + } else { + const trimmedPrefix = prefixEnd.trim(); + const lastWord = trimmedPrefix.split(/\s+/).pop(); + if (lastWord && completion.startsWith(lastWord)) { + completion = completion.slice(lastWord.length); + } else if (completion.startsWith(trimmedPrefix)) { + completion = completion.slice(trimmedPrefix.length); + } + } + } } + // // If completion starts with multiple whitespaces, but the cursor is at the end of the line + // // then it should probably be on a new line + // if ( + // (completion.startsWith(" ") || completion.startsWith("\t")) && + // !prefix.endsWith("\n") && + // (suffix.startsWith("\n") || suffix.trim().length === 0) + // ) { + // completion = "\n" + completion; + // } + // If prefix ends with space and so does completion, then remove the space from completion - if (prefix.endsWith(" ") && completion.startsWith(" ")) { - completion = completion.slice(1); - } - // Qwen often adds an extra space to the start - if (llm.model.toLowerCase().includes("qwen") && completion.startsWith(" ")) { + if (prefix.endsWith(" ") && completion.startsWith(" ")) { completion = completion.slice(1); } diff --git a/core/autocomplete/prefiltering/index.ts b/core/autocomplete/prefiltering/index.ts new file mode 100644 index 0000000000..40ffa5384b --- /dev/null +++ b/core/autocomplete/prefiltering/index.ts @@ -0,0 +1,81 @@ +import path from "node:path"; + +import ignore from "ignore"; + +import { IDE } from "../.."; +import { getBasename } from "../../util"; +import { getConfigJsonPath } from "../../util/paths"; +import { HelperVars } from "../util/HelperVars"; + +async function isDisabledForFile( + currentFilepath: string, + disableInFiles: string[] | undefined, + ide: IDE, +) { + if (disableInFiles) { + // Relative path needed for `ignore` + const workspaceDirs = await ide.getWorkspaceDirs(); + let filepath = currentFilepath; + for (const workspaceDir of workspaceDirs) { + const relativePath = path.relative(workspaceDir, filepath); + const relativePathBase = relativePath.split(path.sep).at(0); + const isInWorkspace = + !path.isAbsolute(relativePath) && relativePathBase !== ".."; + if (isInWorkspace) { + filepath = path.relative(workspaceDir, filepath); + break; + } + } + + // Worst case we can check filetype glob patterns + if (filepath === currentFilepath) { + filepath = getBasename(filepath); + } + + // @ts-ignore + const pattern = ignore.default().add(disableInFiles); + if (pattern.ignores(filepath)) { + return true; + } + } +} + +async function shouldLanguageSpecificPrefilter(helper: HelperVars) { + const line = helper.fileLines[helper.pos.line] ?? ""; + for (const endOfLine of helper.lang.endOfLine) { + if (line.endsWith(endOfLine) && helper.pos.character >= line.length) { + return true; + } + } +} + +export async function shouldPrefilter( + helper: HelperVars, + ide: IDE, +): Promise { + // Allow disabling autocomplete from config.json + if (helper.options.disable) { + return true; + } + + // Check whether we're in the continue config.json file + if (helper.filepath === getConfigJsonPath()) { + return true; + } + + // Check whether autocomplete is disabled for this file + if ( + await isDisabledForFile(helper.filepath, helper.options.disableInFiles, ide) + ) { + return true; + } + + // if ( + // helper.options.transform && + // (await shouldLanguageSpecificPrefilter(helper)) + // ) { + // return true; + // } + + return false; +} diff --git a/core/autocomplete/ranking.ts b/core/autocomplete/ranking.ts deleted file mode 100644 index 48e01d5ff6..0000000000 --- a/core/autocomplete/ranking.ts +++ /dev/null @@ -1,243 +0,0 @@ -import { RangeInFileWithContents } from "../commands/util.js"; -import { Range } from "../index.js"; -import { countTokens } from "../llm/countTokens.js"; - -export type AutocompleteSnippet = RangeInFileWithContents & { - score?: number; -}; - -const rx = /[\s.,\/#!$%\^&\*;:{}=\-_`~()\[\]]/g; -export function getSymbolsForSnippet(snippet: string): Set { - const symbols = snippet - .split(rx) - .map((s) => s.trim()) - .filter((s) => s !== ""); - return new Set(symbols); -} - -/** - * Calculate similarity as number of shared symbols divided by total number of unique symbols between both. - */ -export function jaccardSimilarity(a: string, b: string): number { - const aSet = getSymbolsForSnippet(a); - const bSet = getSymbolsForSnippet(b); - const union = new Set([...aSet, ...bSet]).size; - - // Avoid division by zero - if (union === 0) { - return 0; - } - - let intersection = 0; - for (const symbol of aSet) { - if (bSet.has(symbol)) { - intersection++; - } - } - - return intersection / union; -} - -/** - * Rank code snippets to be used in tab-autocomplete prompt. Returns a sorted version of the snippet array. - */ -export function rankSnippets( - ranges: AutocompleteSnippet[], - windowAroundCursor: string, -): Required[] { - const snippets: Required[] = ranges.map((snippet) => ({ - score: - snippet.score ?? jaccardSimilarity(snippet.contents, windowAroundCursor), - ...snippet, - })); - const uniqueSnippets = deduplicateSnippets(snippets); - return uniqueSnippets.sort((a, b) => a.score - b.score); -} - -/** - * Deduplicate code snippets by merging overlapping ranges into a single range. - */ -export function deduplicateSnippets( - snippets: Required[], -): Required[] { - // Group by file - const fileGroups: { [key: string]: Required[] } = {}; - for (const snippet of snippets) { - if (!fileGroups[snippet.filepath]) { - fileGroups[snippet.filepath] = []; - } - fileGroups[snippet.filepath].push(snippet); - } - - // Merge overlapping ranges - const allRanges = []; - for (const file of Object.keys(fileGroups)) { - allRanges.push(...mergeSnippetsByRange(fileGroups[file])); - } - return allRanges; -} - -function mergeSnippetsByRange( - snippets: Required[], -): Required[] { - if (snippets.length <= 1) { - return snippets; - } - - const sorted = snippets.sort( - (a, b) => a.range.start.line - b.range.start.line, - ); - const merged: Required[] = []; - - while (sorted.length > 0) { - const next = sorted.shift()!; - const last = merged[merged.length - 1]; - if (merged.length > 0 && last.range.end.line >= next.range.start.line) { - // Merge with previous snippet - last.score = Math.max(last.score, next.score); - try { - last.range.end = next.range.end; - } catch (e) { - console.log("Error merging ranges", e); - } - last.contents = mergeOverlappingRangeContents(last, next); - } else { - merged.push(next); - } - } - - return merged; -} - -function mergeOverlappingRangeContents( - first: RangeInFileWithContents, - second: RangeInFileWithContents, -): string { - const firstLines = first.contents.split("\n"); - const numOverlapping = first.range.end.line - second.range.start.line; - return `${firstLines.slice(-numOverlapping).join("\n")}\n${second.contents}`; -} - -/** - * Fill the allowed space with snippets - */ -export function fillPromptWithSnippets( - snippets: Required[], - maxSnippetTokens: number, - modelName: string, -): Required[] { - let tokensRemaining = maxSnippetTokens; - const keptSnippets: Required[] = []; - for (let i = 0; i < snippets.length; i++) { - const snippet = snippets[i]; - const tokenCount = countTokens(snippet.contents, modelName); - if (tokensRemaining - tokenCount >= 0) { - tokensRemaining -= tokenCount; - keptSnippets.push(snippet); - } else { - } - } - - return keptSnippets; -} - -function rangeIntersectionByLines(a: Range, b: Range): Range | null { - const startLine = Math.max(a.start.line, b.start.line); - const endLine = Math.min(a.end.line, b.end.line); - if (startLine >= endLine) { - return null; - } - return { - start: { - line: startLine, - character: 0, - }, - end: { - line: endLine, - character: 0, - }, - }; -} - -/** - * Remove one range from another range, which may lead to returning two disjoint ranges - */ -function rangeDifferenceByLines(orig: Range, remove: Range): Range[] { - if ( - orig.start.line >= remove.start.line && - orig.end.line <= remove.end.line - ) { - // / | | / - return []; - } - if ( - orig.start.line <= remove.start.line && - orig.end.line >= remove.end.line - ) { - // | / / | - // Splits the range - return [ - { - start: orig.start, - end: remove.start, - }, - { - start: remove.end, - end: orig.end, - }, - ]; - } - if ( - orig.start.line >= remove.start.line && - orig.end.line >= remove.end.line - ) { - // \ | / | - return [ - { - start: remove.end, - end: orig.end, - }, - ]; - } - if ( - orig.start.line <= remove.start.line && - orig.end.line <= remove.end.line - ) { - // | / | / - return [ - { - start: orig.start, - end: remove.start, - }, - ]; - } - return [orig]; -} - -export function removeRangeFromSnippets( - snippets: Required[], - filepath: string, - range: Range, -): Required[] { - const finalSnippets: Required[] = []; - for (const snippet of snippets) { - if (snippet.filepath !== filepath) { - finalSnippets.push(snippet); - continue; - } - - const intersection = rangeIntersectionByLines(range, snippet.range); - if (!intersection) { - finalSnippets.push(snippet); - } else { - finalSnippets.push( - ...rangeDifferenceByLines(snippet.range, intersection).map((range) => ({ - ...snippet, - range, - })), - ); - } - } - - return finalSnippets; -} diff --git a/core/autocomplete/recentlyEdited.ts b/core/autocomplete/recentlyEdited.ts deleted file mode 100644 index d0e7817eda..0000000000 --- a/core/autocomplete/recentlyEdited.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { RangeInFile } from "../index.js"; - -export type RecentlyEditedRange = RangeInFile & { - timestamp: number; - lines: string[]; - symbols: Set; -}; - -export function findMatchingRange( - recentlyEditedRanges: RecentlyEditedRange[], - linePrefix: string, -): RecentlyEditedRange | undefined { - return recentlyEditedRanges.find((recentlyEditedRange) => { - return recentlyEditedRange.lines.some((line) => - line.startsWith(linePrefix), - ); - }); -} diff --git a/core/autocomplete/retrieval.ts b/core/autocomplete/retrieval.ts deleted file mode 100644 index 422b377bd6..0000000000 --- a/core/autocomplete/retrieval.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { BranchAndDir, Chunk } from "../index.js"; -import { FullTextSearchCodebaseIndex } from "../indexing/FullTextSearchCodebaseIndex.js"; - -export async function fullTextRetrieve( - prefix: string, - suffix: string, - indexTag: BranchAndDir, -): Promise { - const index = new FullTextSearchCodebaseIndex(); - const searchStrings = prefix.split("\n").slice(-3); - const results: Chunk[] = []; - searchStrings.forEach(async (searchString) => { - const chunks = await index.retrieve( - [indexTag], - searchString, - 3, - undefined, - undefined, - ); - results.push(...chunks); - }); - return results; -} diff --git a/core/autocomplete/services/NearbyDefinitionsService.ts b/core/autocomplete/services/NearbyDefinitionsService.ts deleted file mode 100644 index 12d6f7947f..0000000000 --- a/core/autocomplete/services/NearbyDefinitionsService.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { IDE, Location } from "../.."; -import { LANGUAGES } from "../languages"; -import { getSymbolsForSnippet } from "../ranking"; - -interface FileInfo { - filepath: string; -} - -export class NearbyDefinitionsService { - static N = 10; - - constructor(private readonly ide: IDE) {} - - async getDefinitionsForLine(filepath: string, line: number) { - const lineContent = await this.ide.readRangeInFile(filepath, { - start: { - line, - character: 0, - }, - end: { - line: line + 1, - character: 0, - }, - }); - - // Remove keywords - const lang = LANGUAGES[filepath.split(".").slice(-1)[0]]; - const symbols = Array.from(getSymbolsForSnippet(lineContent)) - .filter((s) => s.length > 0) - .filter((s) => !(lang && lang?.stopWords?.includes(s))); - - return Promise.all( - symbols.map((s) => { - const character = lineContent.indexOf(s); - const pos: Location = { - filepath, - position: { - line, - character, - }, - }; - }), - ); - } -} diff --git a/core/autocomplete/services/RootPathContextService.ts b/core/autocomplete/services/RootPathContextService.ts deleted file mode 100644 index 69426aab81..0000000000 --- a/core/autocomplete/services/RootPathContextService.ts +++ /dev/null @@ -1,129 +0,0 @@ -import { createHash } from "crypto"; -import { LRUCache } from "lru-cache"; -import Parser from "web-tree-sitter"; -import { IDE } from "../.."; -import { getQueryForFile, TSQueryType } from "../../util/treeSitter"; -import { AstPath } from "../ast"; -import { AutocompleteSnippet } from "../ranking"; -import { ImportDefinitionsService } from "./ImportDefinitionsService"; - -export class RootPathContextService { - private cache = new LRUCache({ - max: 100, - }); - - constructor( - private readonly importDefinitionsService: ImportDefinitionsService, - private readonly ide: IDE, - ) {} - - private static getNodeId(node: Parser.SyntaxNode): string { - return `${node.startIndex}`; - } - - private static TYPES_TO_USE = new Set([ - "program", - "function_declaration", - "method_definition", - ]); - - /** - * Key comes from hash of parent key and node type and node id. - */ - private static keyFromNode( - parentKey: string, - astNode: Parser.SyntaxNode, - ): string { - return createHash("sha256") - .update(parentKey) - .update(astNode.type) - .update(RootPathContextService.getNodeId(astNode)) - .digest("hex"); - } - - private async getSnippetsForNode( - filepath: string, - node: Parser.SyntaxNode, - ): Promise { - const snippets: AutocompleteSnippet[] = []; - - let query: Parser.Query | undefined; - switch (node.type) { - case "program": - this.importDefinitionsService.get(filepath); - break; - case "function_declaration": - query = await getQueryForFile( - filepath, - TSQueryType.FunctionDeclaration, - ); - break; - case "method_definition": - query = await getQueryForFile(filepath, TSQueryType.MethodDefinition); - break; - case "function_definition": - query = await getQueryForFile(filepath, TSQueryType.FunctionDefinition); - break; - case "method_declaration": - query = await getQueryForFile(filepath, TSQueryType.MethodDeclaration); - break; - default: - break; - } - - if (!query) { - return snippets; - } - - await Promise.all( - query.matches(node).map(async (match) => { - const startPosition = match.captures[0].node.startPosition; - const endPosition = match.captures[0].node.endPosition; - const definitions = await this.ide.gotoDefinition({ - filepath, - position: { - line: endPosition.row, - character: endPosition.column, - }, - }); - const newSnippets = await Promise.all( - definitions.map(async (def) => ({ - ...def, - contents: await this.ide.readRangeInFile(def.filepath, def.range), - })), - ); - snippets.push(...newSnippets); - }), - ); - - return snippets; - } - - async getContextForPath( - filepath: string, - astPath: AstPath, - // cursorIndex: number, - ): Promise { - const snippets: AutocompleteSnippet[] = []; - - let parentKey = filepath; - for (const astNode of astPath.filter((node) => - RootPathContextService.TYPES_TO_USE.has(node.type), - )) { - const key = RootPathContextService.keyFromNode(parentKey, astNode); - - const foundInCache = this.cache.get(key); - const newSnippets = - foundInCache ?? (await this.getSnippetsForNode(filepath, astNode)); - snippets.push(...newSnippets); - - if (!foundInCache) { - this.cache.set(key, newSnippets); - } - - parentKey = key; - } - - return snippets; - } -} diff --git a/core/autocomplete/slidingWindow.ts b/core/autocomplete/slidingWindow.ts deleted file mode 100644 index 72aa1a5ee6..0000000000 --- a/core/autocomplete/slidingWindow.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { RangeInFileWithContents } from "../commands/util.js"; -import { AutocompleteSnippet, jaccardSimilarity } from "./ranking.js"; - -function* slidingWindow( - content: string, - windowSize: number, -): Generator { - const lines = content.split("\n"); - - let charCount = 0; - let currWindowLines: string[] = []; - for (let i = 0; i < lines.length; i++) { - if (charCount + lines[i].length >= windowSize) { - yield currWindowLines.join("\n"); - currWindowLines = [lines[i]]; - charCount = 0; - } else { - currWindowLines.push(lines[i]); - } - charCount += lines[i].length; - } - - if (currWindowLines.length > 0) { - yield currWindowLines.join("\n"); - } -} - -/** - * Match by similarity over sliding windows of recent documents. - * @param recentFiles - * @param prefix - * @param suffix - */ -export async function slidingWindowMatcher( - recentFiles: RangeInFileWithContents[], - windowAroundCursor: string, - topN: number, - windowSize: number, -): Promise { - // Sorted lowest similarity to highest - const topMatches: Required[] = []; - - for (const { filepath, contents, range } of recentFiles) { - for (const window of slidingWindow(contents, windowSize)) { - const score = jaccardSimilarity(window, windowAroundCursor); - - // Insertion sort - let i = -1; - while (++i < topMatches.length && score > topMatches[i].score) {} - topMatches.splice(i + 1, 0, { filepath, contents, score, range }); - if (topMatches.length > topN) { - topMatches.shift(); - } - } - } - - // TODO: convert the arbitrary window frame to some whole AST node? - return topMatches; -} diff --git a/core/autocomplete/snippets/getAllSnippets.ts b/core/autocomplete/snippets/getAllSnippets.ts new file mode 100644 index 0000000000..30d89e7b94 --- /dev/null +++ b/core/autocomplete/snippets/getAllSnippets.ts @@ -0,0 +1,136 @@ +import { IDE } from "../../index"; +import { ContextRetrievalService } from "../context/ContextRetrievalService"; +import { GetLspDefinitionsFunction } from "../types"; +import { HelperVars } from "../util/HelperVars"; +import { + AutocompleteClipboardSnippet, + AutocompleteCodeSnippet, + AutocompleteDiffSnippet, + AutocompleteSnippetType, +} from "./types"; + +export interface SnippetPayload { + rootPathSnippets: AutocompleteCodeSnippet[]; + importDefinitionSnippets: AutocompleteCodeSnippet[]; + ideSnippets: AutocompleteCodeSnippet[]; + recentlyEditedRangeSnippets: AutocompleteCodeSnippet[]; + diffSnippets: AutocompleteDiffSnippet[]; + clipboardSnippets: AutocompleteClipboardSnippet[]; +} + +function racePromise(promise: Promise): Promise { + const timeoutPromise = new Promise((resolve) => { + setTimeout(() => resolve([]), 100); + }); + + return Promise.race([promise, timeoutPromise]); +} + +// Some IDEs might have special ways of finding snippets (e.g. JetBrains and VS Code have different "LSP-equivalent" systems, +// or they might separately track recently edited ranges) +async function getIdeSnippets( + helper: HelperVars, + ide: IDE, + getDefinitionsFromLsp: GetLspDefinitionsFunction, +): Promise { + const ideSnippets = await getDefinitionsFromLsp( + helper.input.filepath, + helper.fullPrefix + helper.fullSuffix, + helper.fullPrefix.length, + ide, + helper.lang, + ); + + if (helper.options.onlyMyCode) { + const workspaceDirs = await ide.getWorkspaceDirs(); + + return ideSnippets.filter((snippet) => + workspaceDirs.some((dir) => snippet.filepath.startsWith(dir)), + ); + } + + return ideSnippets; +} + +function getSnippetsFromRecentlyEditedRanges( + helper: HelperVars, +): AutocompleteCodeSnippet[] { + if (helper.options.useRecentlyEdited === false) { + return []; + } + + return helper.input.recentlyEditedRanges.map((range) => { + return { + filepath: range.filepath, + content: range.lines.join("\n"), + type: AutocompleteSnippetType.Code, + }; + }); +} + +const getClipboardSnippets = async ( + ide: IDE, +): Promise => { + const content = await ide.getClipboardContent(); + + return [content].map((item) => { + return { + content: item.text, + copiedAt: item.copiedAt, + type: AutocompleteSnippetType.Clipboard, + }; + }); +}; + +const getDiffSnippets = async ( + ide: IDE, +): Promise => { + const diff = await ide.getDiff(true); + + return diff.map((item) => { + return { + content: item, + type: AutocompleteSnippetType.Diff, + }; + }); +}; + +export const getAllSnippets = async ({ + helper, + ide, + getDefinitionsFromLsp, + contextRetrievalService, +}: { + helper: HelperVars; + ide: IDE; + getDefinitionsFromLsp: GetLspDefinitionsFunction; + contextRetrievalService: ContextRetrievalService; +}): Promise => { + const recentlyEditedRangeSnippets = + getSnippetsFromRecentlyEditedRanges(helper); + + const [ + rootPathSnippets, + importDefinitionSnippets, + ideSnippets, + diffSnippets, + clipboardSnippets, + ] = await Promise.all([ + racePromise(contextRetrievalService.getRootPathSnippets(helper)), + racePromise( + contextRetrievalService.getSnippetsFromImportDefinitions(helper), + ), + racePromise(getIdeSnippets(helper, ide, getDefinitionsFromLsp)), + racePromise(getDiffSnippets(ide)), + racePromise(getClipboardSnippets(ide)), + ]); + + return { + rootPathSnippets, + importDefinitionSnippets, + ideSnippets, + recentlyEditedRangeSnippets, + diffSnippets, + clipboardSnippets, + }; +}; diff --git a/core/autocomplete/snippets/index.ts b/core/autocomplete/snippets/index.ts new file mode 100644 index 0000000000..2a9e1a7ced --- /dev/null +++ b/core/autocomplete/snippets/index.ts @@ -0,0 +1 @@ +export * from "./getAllSnippets"; diff --git a/core/autocomplete/snippets/types.ts b/core/autocomplete/snippets/types.ts new file mode 100644 index 0000000000..e43b59862a --- /dev/null +++ b/core/autocomplete/snippets/types.ts @@ -0,0 +1,29 @@ +export enum AutocompleteSnippetType { + Code = "code", + Diff = "diff", + Clipboard = "clipboard", +} + +interface BaseAutocompleteSnippet { + content: string; + type: AutocompleteSnippetType; +} + +export interface AutocompleteCodeSnippet extends BaseAutocompleteSnippet { + filepath: string; + type: AutocompleteSnippetType.Code; +} + +export interface AutocompleteDiffSnippet extends BaseAutocompleteSnippet { + type: AutocompleteSnippetType.Diff; +} + +export interface AutocompleteClipboardSnippet extends BaseAutocompleteSnippet { + type: AutocompleteSnippetType.Clipboard; + copiedAt: string; +} + +export type AutocompleteSnippet = + | AutocompleteCodeSnippet + | AutocompleteDiffSnippet + | AutocompleteClipboardSnippet; diff --git a/core/autocomplete/templates.ts b/core/autocomplete/templating/AutocompleteTemplate.ts similarity index 89% rename from core/autocomplete/templates.ts rename to core/autocomplete/templating/AutocompleteTemplate.ts index cda8b8a2d7..9ade88e46a 100644 --- a/core/autocomplete/templates.ts +++ b/core/autocomplete/templating/AutocompleteTemplate.ts @@ -1,10 +1,14 @@ // Fill in the middle prompts -import { CompletionOptions } from "../index.js"; -import { getLastNPathParts, shortestRelativePaths } from "../util/index.js"; -import { AutocompleteSnippet } from "./ranking.js"; - -interface AutocompleteTemplate { +import { CompletionOptions } from "../../index.js"; +import { getLastNPathParts, shortestRelativePaths } from "../../util/index.js"; +import { + AutocompleteCodeSnippet, + AutocompleteSnippet, + AutocompleteSnippetType, +} from "../snippets/types.js"; + +export interface AutocompleteTemplate { compilePrefixSuffix?: ( prefix: string, suffix: string, @@ -81,13 +85,24 @@ const codestralMultifileFimTemplate: AutocompleteTemplate = { } return [prefix, suffix]; } + const relativePaths = shortestRelativePaths([ - ...snippets.map((snippet) => snippet.filepath), + ...snippets.map((snippet) => + "filepath" in snippet ? snippet.filepath : "Untitled.txt", + ), filepath, ]); + const otherFiles = snippets - .map((snippet, i) => `+++++ ${relativePaths[i]}\n${snippet.contents}`) + .map((snippet, i) => { + if (snippet.type === AutocompleteSnippetType.Diff) { + return snippet.content; + } + + return `+++++ ${relativePaths[i]}\n${snippet.content}`; + }) .join("\n\n"); + return [ `${otherFiles}\n\n+++++ ${ relativePaths[relativePaths.length - 1] @@ -95,14 +110,7 @@ const codestralMultifileFimTemplate: AutocompleteTemplate = { suffix, ]; }, - template: ( - prefix: string, - suffix: string, - filepath: string, - reponame: string, - language: string, - snippets: AutocompleteSnippet[], - ): string => { + template: (prefix: string, suffix: string): string => { return `[SUFFIX]${suffix}[PREFIX]${prefix}`; }, completionOptions: { @@ -140,8 +148,7 @@ const starcoder2FimTemplate: AutocompleteTemplate = { ? "" : `${snippets .map((snippet) => { - return snippet.contents; - // return `${getBasename(snippet.filepath)}\n${snippet.contents}`; + return snippet.content; }) .join("")}`; @@ -187,8 +194,12 @@ const codegeexFimTemplate: AutocompleteTemplate = { filepath: string, reponame: string, language: string, - snippets: AutocompleteSnippet[], + allSnippets: AutocompleteSnippet[], ): string => { + const snippets = allSnippets.filter( + (snippet) => snippet.type === AutocompleteSnippetType.Code, + ) as AutocompleteCodeSnippet[]; + const relativePaths = shortestRelativePaths([ ...snippets.map((snippet) => snippet.filepath), filepath, @@ -200,7 +211,7 @@ const codegeexFimTemplate: AutocompleteTemplate = { return `<|user|>\n${baseTemplate}<|assistant|>\n`; } const references = `###REFERENCE:\n${snippets - .map((snippet, i) => `###PATH:${relativePaths[i]}\n${snippet.contents}\n`) + .map((snippet, i) => `###PATH:${relativePaths[i]}\n${snippet.content}\n`) .join("###REFERENCE:\n")}`; const prompt = `<|user|>\n${references}\n${baseTemplate}<|assistant|>\n`; return prompt; @@ -227,14 +238,7 @@ Fill in the blank to complete the code block. Your response should include only }; const holeFillerTemplate: AutocompleteTemplate = { - template: ( - prefix: string, - suffix: string, - filename: string, - reponame: string, - language: string, - snippets: AutocompleteSnippet[], - ) => { + template: (prefix: string, suffix: string) => { // From https://github.com/VictorTaelin/AI-scripts const SYSTEM_MSG = `You are a HOLE FILLER. You are provided with a file containing holes, formatted as '{{HOLE_NAME}}'. Your TASK is to complete with a string to replace this hole with, inside a XML tag, including context-aware indentation, if needed. All completions MUST be truthful, accurate, well-written and correct. @@ -376,7 +380,8 @@ export function getTemplateForModel(model: string): AutocompleteTemplate { if ( lowerCaseModel.includes("gpt") || lowerCaseModel.includes("davinci-002") || - lowerCaseModel.includes("claude") + lowerCaseModel.includes("claude") || + lowerCaseModel.includes("granite3") ) { return holeFillerTemplate; } diff --git a/core/autocomplete/templating/constructPrefixSuffix.ts b/core/autocomplete/templating/constructPrefixSuffix.ts new file mode 100644 index 0000000000..2bf6fb067a --- /dev/null +++ b/core/autocomplete/templating/constructPrefixSuffix.ts @@ -0,0 +1,43 @@ +import { IDE } from "../.."; +import { getRangeInString } from "../../util/ranges"; +import { languageForFilepath } from "../constants/AutocompleteLanguageInfo"; +import { AutocompleteInput } from "../util/types"; + +/** + * We have to handle a few edge cases in getting the entire prefix/suffix for the current file. + * This is entirely prior to finding snippets from other files + */ +export async function constructInitialPrefixSuffix( + input: AutocompleteInput, + ide: IDE, +): Promise<{ + prefix: string; + suffix: string; +}> { + const lang = languageForFilepath(input.filepath); + + const fileContents = + input.manuallyPassFileContents ?? (await ide.readFile(input.filepath)); + const fileLines = fileContents.split("\n"); + let prefix = + getRangeInString(fileContents, { + start: { line: 0, character: 0 }, + end: input.selectedCompletionInfo?.range.start ?? input.pos, + }) + (input.selectedCompletionInfo?.text ?? ""); + + if (input.injectDetails) { + const lines = prefix.split("\n"); + prefix = `${lines.slice(0, -1).join("\n")}\n${ + lang.singleLineComment + } ${input.injectDetails + .split("\n") + .join(`\n${lang.singleLineComment} `)}\n${lines[lines.length - 1]}`; + } + + const suffix = getRangeInString(fileContents, { + start: input.pos, + end: { line: fileLines.length - 1, character: Number.MAX_SAFE_INTEGER }, + }); + + return { prefix, suffix }; +} diff --git a/core/autocomplete/templating/filtering.ts b/core/autocomplete/templating/filtering.ts new file mode 100644 index 0000000000..a6cdf09572 --- /dev/null +++ b/core/autocomplete/templating/filtering.ts @@ -0,0 +1,75 @@ +import { countTokens } from "../../llm/countTokens"; +import { SnippetPayload } from "../snippets"; +import { + AutocompleteCodeSnippet, + AutocompleteSnippet, +} from "../snippets/types"; +import { HelperVars } from "../util/HelperVars"; +import { isValidSnippet } from "./validation"; + +const getRemainingTokenCount = (helper: HelperVars): number => { + const tokenCount = countTokens(helper.prunedCaretWindow, helper.modelName); + + return helper.options.maxPromptTokens - tokenCount; +}; + +const TOKEN_BUFFER = 10; // We may need extra tokens for snippet description etc. + +/** + * Shuffles an array in place using the Fisher-Yates algorithm. + * @param array The array to shuffle. + * @returns The shuffled array. + */ +const shuffleArray = (array: T[]): T[] => { + for (let i = array.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [array[i], array[j]] = [array[j], array[i]]; + } + return array; +}; + +function filterSnippetsAlreadyInCaretWindow( + snippets: AutocompleteCodeSnippet[], + caretWindow: string, +): AutocompleteCodeSnippet[] { + return snippets.filter( + (s) => s.content.trim() !== "" && !caretWindow.includes(s.content.trim()), + ); +} + +export const getSnippets = ( + helper: HelperVars, + payload: SnippetPayload, +): AutocompleteSnippet[] => { + const snippets = [ + ...payload.diffSnippets, + ...payload.clipboardSnippets, + ...shuffleArray( + filterSnippetsAlreadyInCaretWindow( + [...payload.rootPathSnippets, ...payload.importDefinitionSnippets], + helper.prunedCaretWindow, + ), + ), + ]; + + const finalSnippets = []; + + let remainingTokenCount = getRemainingTokenCount(helper); + + while (remainingTokenCount > 0 && snippets.length > 0) { + const snippet = snippets.shift(); + if (!snippet || !isValidSnippet(snippet)) { + continue; + } + + const snippetSize = + countTokens(snippet.content, helper.modelName) + TOKEN_BUFFER; + + if (remainingTokenCount >= snippetSize) { + finalSnippets.push(snippet); + remainingTokenCount -= snippetSize; + } + } + + return finalSnippets; +}; diff --git a/core/autocomplete/templating/formatting.ts b/core/autocomplete/templating/formatting.ts new file mode 100644 index 0000000000..bdbe56c50f --- /dev/null +++ b/core/autocomplete/templating/formatting.ts @@ -0,0 +1,92 @@ +import { getLastNPathParts } from "../../util"; +import { + AutocompleteClipboardSnippet, + AutocompleteCodeSnippet, + AutocompleteDiffSnippet, + AutocompleteSnippet, + AutocompleteSnippetType, +} from "../snippets/types"; +import { HelperVars } from "../util/HelperVars"; + +const getCommentMark = (helper: HelperVars) => { + return helper.lang.singleLineComment; +}; + +const addCommentMarks = (text: string, helper: HelperVars) => { + const commentMark = getCommentMark(helper); + const lines = [ + ...text + .trim() + .split("\n") + .map((line) => `${commentMark} ${line}`), + ]; + + return lines.join("\n"); +}; + +const formatClipboardSnippet = ( + snippet: AutocompleteClipboardSnippet, +): AutocompleteCodeSnippet => { + return formatCodeSnippet({ + filepath: "Untitled.txt", + content: snippet.content, + type: AutocompleteSnippetType.Code, + }); +}; + +const formatCodeSnippet = ( + snippet: AutocompleteCodeSnippet, +): AutocompleteCodeSnippet => { + return { + ...snippet, + content: `Path: ${getLastNPathParts(snippet.filepath, 2)}\n${snippet.content}`, + }; +}; + +const formatDiffSnippet = ( + snippet: AutocompleteDiffSnippet, +): AutocompleteDiffSnippet => { + return snippet; +}; + +const getCurrentFilepath = (helper: HelperVars) => { + return getLastNPathParts(helper.filepath, 2); +}; + +const commentifySnippet = ( + helper: HelperVars, + snippet: AutocompleteSnippet, +): AutocompleteSnippet => { + return { + ...snippet, + content: addCommentMarks(snippet.content, helper), + }; +}; + +export const formatSnippets = ( + helper: HelperVars, + snippets: AutocompleteSnippet[], +): string => { + const currentFilepathComment = addCommentMarks( + getCurrentFilepath(helper), + helper, + ); + + return ( + snippets + .map((snippet) => { + switch (snippet.type) { + case AutocompleteSnippetType.Code: + return formatCodeSnippet(snippet); + case AutocompleteSnippetType.Diff: + return formatDiffSnippet(snippet); + case AutocompleteSnippetType.Clipboard: + return formatClipboardSnippet(snippet); + } + }) + .map((item) => { + return commentifySnippet(helper, item).content; + }) + .join("\n") + `\n${currentFilepathComment}` + ); +}; diff --git a/core/autocomplete/templating/getStopTokens.ts b/core/autocomplete/templating/getStopTokens.ts new file mode 100644 index 0000000000..ece0a73b4e --- /dev/null +++ b/core/autocomplete/templating/getStopTokens.ts @@ -0,0 +1,32 @@ +import { CompletionOptions } from "../.."; +import { AutocompleteLanguageInfo } from "../constants/AutocompleteLanguageInfo"; + +const DOUBLE_NEWLINE = "\n\n"; +const WINDOWS_DOUBLE_NEWLINE = "\r\n\r\n"; +// TODO: Do we want to stop completions when reaching a `/src/` string? +const SRC_DIRECTORY = "/src/"; +// Starcoder2 tends to output artifacts starting with the letter "t" +const STARCODER2_T_ARTIFACTS = ["t.", "\nt", ""]; +const PYTHON_ENCODING = "#- coding: utf-8"; +const CODE_BLOCK_END = "```"; + +// const multilineStops: string[] = [DOUBLE_NEWLINE, WINDOWS_DOUBLE_NEWLINE]; +const commonStops = [SRC_DIRECTORY, PYTHON_ENCODING, CODE_BLOCK_END]; + +export function getStopTokens( + completionOptions: Partial | undefined, + lang: AutocompleteLanguageInfo, + model: string, +): string[] { + const stopTokens = [ + ...(completionOptions?.stop || []), + // ...multilineStops, + ...commonStops, + ...(model.toLowerCase().includes("starcoder2") + ? STARCODER2_T_ARTIFACTS + : []), + // ...lang.topLevelKeywords.map((word) => `\n${word}`), + ]; + + return stopTokens; +} diff --git a/core/autocomplete/templating/index.ts b/core/autocomplete/templating/index.ts new file mode 100644 index 0000000000..8aaa982487 --- /dev/null +++ b/core/autocomplete/templating/index.ts @@ -0,0 +1,123 @@ +import Handlebars from "handlebars"; + +import { CompletionOptions } from "../.."; +import { getBasename } from "../../util"; +import { AutocompleteLanguageInfo } from "../constants/AutocompleteLanguageInfo"; +import { HelperVars } from "../util/HelperVars"; + +import { + AutocompleteTemplate, + getTemplateForModel, +} from "./AutocompleteTemplate"; +import { getStopTokens } from "./getStopTokens"; +import { SnippetPayload } from "../snippets"; +import { formatSnippets } from "./formatting"; +import { getSnippets } from "./filtering"; + +function getTemplate(helper: HelperVars): AutocompleteTemplate { + if (helper.options.template) { + return { + template: helper.options.template, + completionOptions: {}, + compilePrefixSuffix: undefined, + }; + } + return getTemplateForModel(helper.modelName); +} + +function renderStringTemplate( + template: string, + prefix: string, + suffix: string, + lang: AutocompleteLanguageInfo, + filepath: string, + reponame: string, +) { + const filename = getBasename(filepath); + const compiledTemplate = Handlebars.compile(template); + + return compiledTemplate({ + prefix, + suffix, + filename, + reponame, + language: lang.name, + }); +} + +export function renderPrompt({ + snippetPayload, + workspaceDirs, + helper, +}: { + snippetPayload: SnippetPayload; + workspaceDirs: string[]; + helper: HelperVars; +}): { + prompt: string; + prefix: string; + suffix: string; + completionOptions: Partial | undefined; +} { + // If prefix is manually passed + let prefix = helper.input.manuallyPassPrefix || helper.prunedPrefix; + let suffix = helper.input.manuallyPassPrefix ? "" : helper.prunedSuffix; + + const reponame = getBasename(workspaceDirs[0] ?? "myproject"); + + const { template, compilePrefixSuffix, completionOptions } = + getTemplate(helper); + + const snippets = getSnippets(helper, snippetPayload); + + // Some models have prompts that need two passes. This lets us pass the compiled prefix/suffix + // into either the 2nd template to generate a raw string, or to pass prefix, suffix to a FIM endpoint + if (compilePrefixSuffix) { + [prefix, suffix] = compilePrefixSuffix( + prefix, + suffix, + helper.filepath, + reponame, + snippets, + ); + } else { + const formattedSnippets = formatSnippets(helper, snippets); + prefix = [formattedSnippets, prefix].join("\n"); + } + + const prompt = + // Templates can be passed as a Handlebars template string or a function + typeof template === "string" + ? renderStringTemplate( + template, + prefix, + suffix, + helper.lang, + helper.filepath, + reponame, + ) + : template( + prefix, + suffix, + helper.filepath, + reponame, + helper.lang.name, + snippets, + ); + + const stopTokens = getStopTokens( + completionOptions, + helper.lang, + helper.modelName, + ); + + return { + prompt, + prefix, + suffix, + completionOptions: { + ...completionOptions, + stop: stopTokens, + }, + }; +} diff --git a/core/autocomplete/templating/validation.ts b/core/autocomplete/templating/validation.ts new file mode 100644 index 0000000000..dad55dc5cc --- /dev/null +++ b/core/autocomplete/templating/validation.ts @@ -0,0 +1,29 @@ +import { + AutocompleteClipboardSnippet, + AutocompleteSnippet, + AutocompleteSnippetType, +} from "../snippets/types"; + +const MAX_CLIPBOARD_AGE = 5 * 60 * 1000; + +const isValidClipboardSnippet = ( + snippet: AutocompleteClipboardSnippet, +): boolean => { + const currDate = new Date(); + + const isTooOld = + currDate.getTime() - new Date(snippet.copiedAt).getTime() > + MAX_CLIPBOARD_AGE; + + return !isTooOld; +}; + +export const isValidSnippet = (snippet: AutocompleteSnippet): boolean => { + if (snippet.content.trim() === "") return false; + + if (snippet.type === AutocompleteSnippetType.Clipboard) { + return isValidClipboardSnippet(snippet); + } + + return true; +}; diff --git a/core/autocomplete/types.ts b/core/autocomplete/types.ts new file mode 100644 index 0000000000..23f7625566 --- /dev/null +++ b/core/autocomplete/types.ts @@ -0,0 +1,20 @@ +import { IDE, RangeInFileWithContents } from "../index"; +import { AutocompleteLanguageInfo } from "./constants/AutocompleteLanguageInfo"; +import { AutocompleteCodeSnippet } from "./snippets/types"; + +/** + * @deprecated This type should be removed in the future or renamed. + * We have a new interface called AutocompleteSnippet which is more + * general. + */ +export type AutocompleteSnippetDeprecated = RangeInFileWithContents & { + score?: number; +}; + +export type GetLspDefinitionsFunction = ( + filepath: string, + contents: string, + cursorIndex: number, + ide: IDE, + lang: AutocompleteLanguageInfo, +) => Promise; diff --git a/core/autocomplete/util.ts b/core/autocomplete/util.ts deleted file mode 100644 index d07658d17e..0000000000 --- a/core/autocomplete/util.ts +++ /dev/null @@ -1,142 +0,0 @@ -export class ListenableGenerator { - private _source: AsyncGenerator; - private _buffer: T[] = []; - private _listeners: Set<(value: T) => void> = new Set(); - private _isEnded = false; - - constructor( - source: AsyncGenerator, - private readonly onError: (e: any) => void, - ) { - this._source = source; - this._start(); - } - - public cancel() { - this._isEnded = true; - } - - private async _start() { - try { - for await (const value of this._source) { - if (this._isEnded) { - break; - } - this._buffer.push(value); - for (const listener of this._listeners) { - listener(value); - } - } - } catch (e) { - this.onError(e); - } finally { - this._isEnded = true; - for (const listener of this._listeners) { - listener(null as any); - } - } - } - - listen(listener: (value: T) => void) { - this._listeners.add(listener); - for (const value of this._buffer) { - listener(value); - } - if (this._isEnded) { - listener(null as any); - } - } - - async *tee(): AsyncGenerator { - try { - let i = 0; - while (i < this._buffer.length) { - yield this._buffer[i++]; - } - while (!this._isEnded) { - let resolve: (value: any) => void; - const promise = new Promise((res) => { - resolve = res; - this._listeners.add(resolve!); - }); - await promise; - this._listeners.delete(resolve!); - - // Possible timing caused something to slip in between - // timers so we iterate over the buffer - while (i < this._buffer.length) { - yield this._buffer[i++]; - } - } - } finally { - // this._listeners.delete(resolve!); - } - } -} - -export class GeneratorReuseManager { - currentGenerator: ListenableGenerator | undefined; - pendingGeneratorPrefix: string | undefined; - pendingCompletion = ""; - - constructor(private readonly onError: (err: any) => void) {} - - private _createListenableGenerator( - gen: AsyncGenerator, - prefix: string, - ) { - this.currentGenerator?.cancel(); - - const listenableGen = new ListenableGenerator(gen, this.onError); - listenableGen.listen((chunk) => (this.pendingCompletion += chunk ?? "")); - - this.pendingGeneratorPrefix = prefix; - this.pendingCompletion = ""; - this.currentGenerator = listenableGen; - } - - async *getGenerator( - prefix: string, - newGenerator: () => AsyncGenerator, - multiline: boolean, - ): AsyncGenerator { - // Check if current can be reused - if ( - !( - this.currentGenerator && - this.pendingGeneratorPrefix && - (this.pendingGeneratorPrefix + this.pendingCompletion).startsWith( - prefix, - ) && - // for e.g. backspace - this.pendingGeneratorPrefix?.length <= prefix?.length - ) - ) { - // Create a wrapper over the current generator to fix the prompt - this._createListenableGenerator(newGenerator(), prefix); - } - - let alreadyTyped = prefix.slice(this.pendingGeneratorPrefix?.length) || ""; - for await (let chunk of this.currentGenerator?.tee() ?? []) { - if (!chunk) { - continue; - } - while (chunk.length && alreadyTyped.length) { - if (chunk[0] === alreadyTyped[0]) { - alreadyTyped = alreadyTyped.slice(1); - chunk = chunk.slice(1); - } else { - break; - } - } - - const newLineIndex = chunk.indexOf("\n"); - if (multiline || newLineIndex === -1) { - yield chunk; - } else { - yield chunk.slice(0, newLineIndex); - break; - } - } - } -} diff --git a/core/autocomplete/util/AutocompleteDebouncer.ts b/core/autocomplete/util/AutocompleteDebouncer.ts new file mode 100644 index 0000000000..f8bcc474f3 --- /dev/null +++ b/core/autocomplete/util/AutocompleteDebouncer.ts @@ -0,0 +1,32 @@ +import { v4 as uuidv4 } from "uuid"; +export class AutocompleteDebouncer { + private debounceTimeout: NodeJS.Timeout | undefined = undefined; + private debouncing = false; + private lastUUID: string | undefined = undefined; + + async delayAndShouldDebounce(debounceDelay: number): Promise { + // Debounce + const uuid = uuidv4(); + this.lastUUID = uuid; + + // Debounce + if (this.debouncing) { + this.debounceTimeout?.refresh(); + const lastUUID = await new Promise((resolve) => + setTimeout(() => { + resolve(this.lastUUID); + }, debounceDelay), + ); + if (uuid !== lastUUID) { + return true; + } + } else { + this.debouncing = true; + this.debounceTimeout = setTimeout(async () => { + this.debouncing = false; + }, debounceDelay); + } + + return false; + } +} diff --git a/core/autocomplete/util/AutocompleteLoggingService.ts b/core/autocomplete/util/AutocompleteLoggingService.ts new file mode 100644 index 0000000000..35e0ead644 --- /dev/null +++ b/core/autocomplete/util/AutocompleteLoggingService.ts @@ -0,0 +1,119 @@ +import { logDevData } from "../../util/devdata"; +import { COUNT_COMPLETION_REJECTED_AFTER } from "../../util/parameters"; +import { Telemetry } from "../../util/posthog"; + +import { AutocompleteOutcome } from "./types"; + +export class AutocompleteLoggingService { + // Key is completionId + private _abortControllers = new Map(); + private _logRejectionTimeouts = new Map(); + private _outcomes = new Map(); + _lastDisplayedCompletion: { id: string; displayedAt: number } | undefined = + undefined; + + public createAbortController(completionId: string): AbortController { + const abortController = new AbortController(); + this._abortControllers.set(completionId, abortController); + return abortController; + } + + public deleteAbortController(completionId: string) { + this._abortControllers.delete(completionId); + } + + public cancel() { + this._abortControllers.forEach((abortController, id) => { + abortController.abort(); + }); + this._abortControllers.clear(); + } + + public accept(completionId: string): AutocompleteOutcome | undefined { + if (this._logRejectionTimeouts.has(completionId)) { + clearTimeout(this._logRejectionTimeouts.get(completionId)); + this._logRejectionTimeouts.delete(completionId); + } + + if (this._outcomes.has(completionId)) { + const outcome = this._outcomes.get(completionId)!; + outcome.accepted = true; + this.logAutocompleteOutcome(outcome); + this._outcomes.delete(completionId); + return outcome; + } + } + + public cancelRejectionTimeout(completionId: string) { + if (this._logRejectionTimeouts.has(completionId)) { + clearTimeout(this._logRejectionTimeouts.get(completionId)!); + this._logRejectionTimeouts.delete(completionId); + } + + if (this._outcomes.has(completionId)) { + this._outcomes.delete(completionId); + } + } + + public markDisplayed(completionId: string, outcome: AutocompleteOutcome) { + const logRejectionTimeout = setTimeout(() => { + // Wait 10 seconds, then assume it wasn't accepted + outcome.accepted = false; + this.logAutocompleteOutcome(outcome); + this._logRejectionTimeouts.delete(completionId); + }, COUNT_COMPLETION_REJECTED_AFTER); + this._outcomes.set(completionId, outcome); + this._logRejectionTimeouts.set(completionId, logRejectionTimeout); + + // If the previously displayed completion is still waiting for rejection, + // and this one is a continuation of that (the outcome.completion is the same modulo prefix) + // then we should cancel the rejection timeout + const previous = this._lastDisplayedCompletion; + const now = Date.now(); + if (previous && this._logRejectionTimeouts.has(previous.id)) { + const previousOutcome = this._outcomes.get(previous.id); + const c1 = previousOutcome?.completion.split("\n")[0] ?? ""; + const c2 = outcome.completion.split("\n")[0]; + if ( + previousOutcome && + (c1.endsWith(c2) || + c2.endsWith(c1) || + c1.startsWith(c2) || + c2.startsWith(c1)) + ) { + this.cancelRejectionTimeout(previous.id); + } else if (now - previous.displayedAt < 500) { + // If a completion isn't shown for more than + this.cancelRejectionTimeout(previous.id); + } + } + + this._lastDisplayedCompletion = { + id: completionId, + displayedAt: now, + }; + } + + private logAutocompleteOutcome(outcome: AutocompleteOutcome) { + logDevData("autocomplete", outcome); + const { prompt, completion, prefix, suffix, ...restOfOutcome } = outcome; + void Telemetry.capture( + "autocomplete", + { + accepted: restOfOutcome.accepted, + cacheHit: restOfOutcome.cacheHit, + completionId: restOfOutcome.completionId, + completionOptions: restOfOutcome.completionOptions, + debounceDelay: restOfOutcome.debounceDelay, + fileExtension: restOfOutcome.filepath.split(".")?.slice(-1)[0], + maxPromptTokens: restOfOutcome.maxPromptTokens, + modelName: restOfOutcome.modelName, + modelProvider: restOfOutcome.modelProvider, + multilineCompletions: restOfOutcome.multilineCompletions, + time: restOfOutcome.time, + useRecentlyEdited: restOfOutcome.useRecentlyEdited, + }, + true, + ); + } +} diff --git a/core/autocomplete/cache.ts b/core/autocomplete/util/AutocompleteLruCache.ts similarity index 91% rename from core/autocomplete/cache.ts rename to core/autocomplete/util/AutocompleteLruCache.ts index 5005480bf3..a93b23c5a6 100644 --- a/core/autocomplete/cache.ts +++ b/core/autocomplete/util/AutocompleteLruCache.ts @@ -1,18 +1,18 @@ import { Mutex } from "async-mutex"; import { open } from "sqlite"; import sqlite3 from "sqlite3"; -import { DatabaseConnection } from "../indexing/refreshIndex.js"; -import { getTabAutocompleteCacheSqlitePath } from "../util/paths.js"; + +import { + DatabaseConnection, + truncateSqliteLikePattern, +} from "../../indexing/refreshIndex.js"; +import { getTabAutocompleteCacheSqlitePath } from "../../util/paths.js"; export class AutocompleteLruCache { private static capacity = 1000; private mutex = new Mutex(); - db: DatabaseConnection; - - constructor(db: DatabaseConnection) { - this.db = db; - } + constructor(private db: DatabaseConnection) {} static async get(): Promise { const db = await open({ @@ -41,7 +41,7 @@ export class AutocompleteLruCache { // Have to make sure we take the key with shortest length const result = await this.db.get( "SELECT key, value FROM cache WHERE ? LIKE key || '%' ORDER BY LENGTH(key) DESC LIMIT 1", - prefix, + truncateSqliteLikePattern(prefix), ); // Validate that the cached compeltion is a valid completion for the prefix diff --git a/core/autocomplete/util/HelperVars.ts b/core/autocomplete/util/HelperVars.ts new file mode 100644 index 0000000000..0470384936 --- /dev/null +++ b/core/autocomplete/util/HelperVars.ts @@ -0,0 +1,176 @@ +import { IDE, TabAutocompleteOptions } from "../.."; +import { + countTokens, + pruneLinesFromBottom, + pruneLinesFromTop, +} from "../../llm/countTokens"; +import { + AutocompleteLanguageInfo, + languageForFilepath, +} from "../constants/AutocompleteLanguageInfo"; +import { constructInitialPrefixSuffix } from "../templating/constructPrefixSuffix"; + +import { AstPath, getAst, getTreePathAtCursor } from "./ast"; +import { AutocompleteInput } from "./types"; + +/** + * A collection of variables that are often accessed throughout the autocomplete pipeline + * It's noisy to re-calculate all the time or inject them into each function + */ +export class HelperVars { + lang: AutocompleteLanguageInfo; + treePath: AstPath | undefined; + + private _fileContents: string | undefined; + private _fileLines: string[] | undefined; + private _fullPrefix: string | undefined; + private _fullSuffix: string | undefined; + private _prunedPrefix: string | undefined; + private _prunedSuffix: string | undefined; + + private constructor( + public readonly input: AutocompleteInput, + public readonly options: TabAutocompleteOptions, + public readonly modelName: string, + private readonly ide: IDE, + ) { + this.lang = languageForFilepath(input.filepath); + } + + private async init() { + // Don't do anything if already initialized + if (this._fileContents !== undefined) { + return; + } + + this._fileContents = + this.input.manuallyPassFileContents ?? + (await this.ide.readFile(this.filepath)); + + this._fileLines = this._fileContents.split("\n"); + + // Construct full prefix/suffix (a few edge cases handled in here) + const { prefix: fullPrefix, suffix: fullSuffix } = + await constructInitialPrefixSuffix(this.input, this.ide); + this._fullPrefix = fullPrefix; + this._fullSuffix = fullSuffix; + + const { prunedPrefix, prunedSuffix } = this.prunePrefixSuffix(); + this._prunedPrefix = prunedPrefix; + this._prunedSuffix = prunedSuffix; + + try { + const ast = await getAst(this.filepath, fullPrefix + fullSuffix); + if (ast) { + this.treePath = await getTreePathAtCursor(ast, fullPrefix.length); + } + } catch (e) { + console.error("Failed to parse AST", e); + } + } + + static async create( + input: AutocompleteInput, + options: TabAutocompleteOptions, + modelName: string, + ide: IDE, + ): Promise { + const instance = new HelperVars(input, options, modelName, ide); + await instance.init(); + return instance; + } + + prunePrefixSuffix() { + // Construct basic prefix + const maxPrefixTokens = + this.options.maxPromptTokens * this.options.prefixPercentage; + const prunedPrefix = pruneLinesFromTop( + this.fullPrefix, + maxPrefixTokens, + this.modelName, + ); + + // Construct suffix + const maxSuffixTokens = Math.min( + this.options.maxPromptTokens - countTokens(prunedPrefix, this.modelName), + this.options.maxSuffixPercentage * this.options.maxPromptTokens, + ); + const prunedSuffix = pruneLinesFromBottom( + this.fullSuffix, + maxSuffixTokens, + this.modelName, + ); + + return { + prunedPrefix, + prunedSuffix, + }; + } + + // Fast access + get filepath() { + return this.input.filepath; + } + get pos() { + return this.input.pos; + } + + get prunedCaretWindow() { + return this.prunedPrefix + this.prunedSuffix; + } + + // Getters for lazy access + get fileContents(): string { + if (this._fileContents === undefined) { + throw new Error( + "HelperVars must be initialized before accessing fileContents", + ); + } + return this._fileContents; + } + + get fileLines(): string[] { + if (this._fileLines === undefined) { + throw new Error( + "HelperVars must be initialized before accessing fileLines", + ); + } + return this._fileLines; + } + + get fullPrefix(): string { + if (this._fullPrefix === undefined) { + throw new Error( + "HelperVars must be initialized before accessing fullPrefix", + ); + } + return this._fullPrefix; + } + + get fullSuffix(): string { + if (this._fullSuffix === undefined) { + throw new Error( + "HelperVars must be initialized before accessing fullSuffix", + ); + } + return this._fullSuffix; + } + + get prunedPrefix(): string { + if (this._prunedPrefix === undefined) { + throw new Error( + "HelperVars must be initialized before accessing prunedPrefix", + ); + } + return this._prunedPrefix; + } + + get prunedSuffix(): string { + if (this._prunedSuffix === undefined) { + throw new Error( + "HelperVars must be initialized before accessing prunedSuffix", + ); + } + return this._prunedSuffix; + } +} diff --git a/core/autocomplete/ast.ts b/core/autocomplete/util/ast.ts similarity index 94% rename from core/autocomplete/ast.ts rename to core/autocomplete/util/ast.ts index 139b22dbf3..241c42fb9c 100644 --- a/core/autocomplete/ast.ts +++ b/core/autocomplete/util/ast.ts @@ -1,6 +1,7 @@ import Parser from "web-tree-sitter"; -import { RangeInFileWithContents } from "../commands/util.js"; -import { getParserForFile } from "../util/treeSitter.js"; + +import { RangeInFileWithContents } from "../../"; +import { getParserForFile } from "../../util/treeSitter"; export type AstPath = Parser.SyntaxNode[]; diff --git a/core/autocomplete/util/types.ts b/core/autocomplete/util/types.ts new file mode 100644 index 0000000000..d54d57bd2f --- /dev/null +++ b/core/autocomplete/util/types.ts @@ -0,0 +1,48 @@ +import { + Position, + Range, + RangeInFile, + RangeInFileWithContents, + TabAutocompleteOptions, +} from "../.."; + +export type RecentlyEditedRange = RangeInFile & { + timestamp: number; + lines: string[]; + symbols: Set; +}; + +export interface AutocompleteInput { + completionId: string; + filepath: string; + pos: Position; + recentlyEditedFiles: RangeInFileWithContents[]; + recentlyEditedRanges: RecentlyEditedRange[]; + // Used for notebook files + manuallyPassFileContents?: string; + // Used for VS Code git commit input box + manuallyPassPrefix?: string; + selectedCompletionInfo?: { + text: string; + range: Range; + }; + injectDetails?: string; +} + +export interface AutocompleteOutcome extends TabAutocompleteOptions { + accepted?: boolean; + time: number; + prefix: string; + suffix: string; + prompt: string; + completion: string; + modelProvider: string; + modelName: string; + completionOptions: any; + cacheHit: boolean; + filepath: string; + gitRepo?: string; + completionId: string; + uniqueId: string; + timestamp: number; +} diff --git a/core/commands/index.ts b/core/commands/index.ts index fd075c6ff0..fef7d533d5 100644 --- a/core/commands/index.ts +++ b/core/commands/index.ts @@ -1,6 +1,7 @@ import { CustomCommand, SlashCommand, SlashCommandDescription } from "../"; -import { stripImages } from "../llm/images"; -import { renderTemplatedString } from "../promptFiles/renderTemplatedString"; +import { renderTemplatedString } from "../promptFiles/v1/renderTemplatedString"; +import { renderChatMessage } from "../util/messageContent"; + import SlashCommands from "./slash"; export function slashFromCustomCommand( @@ -28,7 +29,8 @@ export function slashFromCustomCommand( const messages = [...history]; // Find the last chat message with this slash command and replace it with the user input for (let i = messages.length - 1; i >= 0; i--) { - const { role, content } = messages[i]; + const message = messages[i]; + const { role, content } = message; if (role !== "user") { continue; } @@ -40,7 +42,7 @@ export function slashFromCustomCommand( ) ) { messages[i] = { - ...messages[i], + ...message, content: content.map((part) => { return part.text?.startsWith(`/${customCommand.name}`) ? { ...part, text: promptUserInput } @@ -52,13 +54,16 @@ export function slashFromCustomCommand( typeof content === "string" && content.startsWith(`/${customCommand.name}`) ) { - messages[i] = { ...messages[i], content: promptUserInput }; + messages[i] = { ...message, content: promptUserInput }; break; } } - for await (const chunk of llm.streamChat(messages)) { - yield stripImages(chunk.content); + for await (const chunk of llm.streamChat( + messages, + new AbortController().signal, + )) { + yield renderChatMessage(chunk); } }, }; @@ -67,7 +72,7 @@ export function slashFromCustomCommand( export function slashCommandFromDescription( desc: SlashCommandDescription, ): SlashCommand | undefined { - const cmd = SlashCommands.find((cmd) => cmd.name === desc.name); + const cmd = SlashCommands.find((cmd) => cmd?.name === desc.name); if (!cmd) { return undefined; } diff --git a/core/commands/slash/cmd.ts b/core/commands/slash/cmd.ts index 00ba5b0dee..1c3968172e 100644 --- a/core/commands/slash/cmd.ts +++ b/core/commands/slash/cmd.ts @@ -24,7 +24,9 @@ const GenerateTerminalCommand: SlashCommand = { "${input}" -Please write a shell command that will do what the user requested. Your output should consist of only the command itself, without any explanation or example output. Do not use any newlines. Only output the command that when inserted into the terminal will do precisely what was requested. Here is the command:`); +Please write a shell command that will do what the user requested. Your output should consist of only the command itself, without any explanation or example output. Do not use any newlines. Only output the command that when inserted into the terminal will do precisely what was requested. Here is the command:`, + new AbortController().signal + ); const lines = streamLines(gen); let cmd = ""; diff --git a/core/commands/slash/code-stats.ts b/core/commands/slash/code-stats.ts new file mode 100644 index 0000000000..293d485550 --- /dev/null +++ b/core/commands/slash/code-stats.ts @@ -0,0 +1,130 @@ +import { IDE, SlashCommand } from "../.."; +import * as fs from "fs/promises"; +import * as path from "path"; +import ignore from "ignore"; +import { + defaultIgnoreDir, + defaultIgnoreFile, + gitIgArrayFromFile, +} from "../../indexing/ignore"; +import { renderChatMessage } from "../../util/messageContent"; + +const LANGUAGE_DEP_MGMT_FILENAMES = [ + "package.json", // JavaScript (Node.js) + "requirements.txt", // Python + "Gemfile", // Ruby + "pom.xml", // Java (Maven) + "build.gradle", // Java (Gradle) + "composer.json", // PHP + "Cargo.toml", // Rust + "go.mod", // Go + "packages.config", // C# (.NET) + "*.csproj", // C# (.NET Core) + "pubspec.yaml", // Dart + "Project.toml", // Julia + "mix.exs", // Elixir + "rebar.config", // Erlang + "shard.yml", // Crystal + "Package.swift", // Swift + "dependencies.gradle", // Kotlin (when using Gradle) + "Podfile", // Objective-C/Swift (CocoaPods) + "*.cabal", // Haskell + "dub.json", // D +]; + +const MAX_EXPLORE_DEPTH = 2; + +const CreateCodeStatsCommand: SlashCommand = { + name: "code-stats", + description: "Generate stats of the codebase.", + run: async function* ({ llm, ide, input }) { + const [workspaceDir] = await ide.getWorkspaceDirs(); + + const context = await gatherProjectContext(workspaceDir, ide); + const prompt = createCodeStatsPrompt(context, input); + + for await (const chunk of llm.streamChat( + [{ role: "user", content: prompt }], + new AbortController().signal, + )) { + yield renderChatMessage(chunk); + } + }, +}; + +async function getEntriesFilteredByIgnore(dir: string, ide: IDE) { + const entries = await fs.readdir(dir, { withFileTypes: true }); + + let ig = ignore().add(defaultIgnoreDir).add(defaultIgnoreFile); + + const gitIgnorePath = path.join(dir, ".gitignore"); + + const hasIgnoreFile = await fs + .access(gitIgnorePath) + .then(() => true) + .catch(() => false); + + if (hasIgnoreFile) { + const gitIgnore = await ide.readFile(gitIgnorePath); + const igPatterns = gitIgArrayFromFile(gitIgnore); + + ig = ig.add(igPatterns); + } + + const filteredEntries = entries.filter((entry) => !ig.ignores(entry.name)); + + return filteredEntries; +} + +async function gatherProjectContext( + workspaceDir: string, + ide: IDE, +): Promise { + let context = ""; + + async function exploreDirectory(dir: string, currentDepth: number = 0) { + if (currentDepth > MAX_EXPLORE_DEPTH) { + return; + } + + const entries = await getEntriesFilteredByIgnore(dir, ide); + + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + const relativePath = path.relative(workspaceDir, fullPath); + + if (entry.isDirectory()) { + context += `\nFolder: ${relativePath}\n`; + await exploreDirectory(fullPath, currentDepth + 1); + } else { + if (entry.name.toLowerCase() === "readme.md") { + const content = await fs.readFile(fullPath, "utf-8"); + context += `README for ${relativePath}:\n${content}\n\n`; + } else if (LANGUAGE_DEP_MGMT_FILENAMES.includes(entry.name)) { + const content = await fs.readFile(fullPath, "utf-8"); + context += `${entry.name} for ${relativePath}:\n${content}\n\n`; + } + } + } + } + + await exploreDirectory(workspaceDir); + + return context; +} + +function createCodeStatsPrompt(context: string, input: string): string { + return ` + Please help me generate detailed statistics about the current project. The goal is to analyze the codebase and provide the following information: + + Language Usage: Number of files and total lines of code (LOC) for each programming language used in the project. + File Count: The total number of files in the project. + Total LOC: The combined lines of code for all files in the project. + + Please format the statistics in a clean and organized way + + If additional details like the largest files, smallest files, or a summary of extensions used can be included, feel free to add them!" + `; +} + +export default CreateCodeStatsCommand; diff --git a/core/commands/slash/comment.ts b/core/commands/slash/comment.ts deleted file mode 100644 index e3b601c04d..0000000000 --- a/core/commands/slash/comment.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { SlashCommand } from "../../"; -import EditSlashCommand from "./edit"; - -const CommentSlashCommand: SlashCommand = { - name: "comment", - description: "Write comments for highlighted code", - run: async function* (sdk) { - for await (const update of EditSlashCommand.run({ - ...sdk, - input: - "Write comments for this code. Do not change anything about the code itself.", - })) { - yield update; - } - }, -}; - -export default CommentSlashCommand; diff --git a/core/commands/slash/commit.ts b/core/commands/slash/commit.ts index 899222998d..c2d09adba9 100644 --- a/core/commands/slash/commit.ts +++ b/core/commands/slash/commit.ts @@ -1,22 +1,24 @@ import { SlashCommand } from "../../index.js"; -import { stripImages } from "../../llm/images.js"; +import { renderChatMessage } from "../../util/messageContent.js"; const CommitMessageCommand: SlashCommand = { name: "commit", description: "Generate a commit message for current changes", - run: async function* ({ ide, llm, input }) { - const diff = await ide.getDiff(); + run: async function* ({ ide, llm, params }) { + const includeUnstaged = params?.includeUnstaged ?? true; + const diff = await ide.getDiff(includeUnstaged); - if (!diff || diff.trim() === "") { + if (diff.length === 0) { yield "No changes detected. Make sure you are in a git repository with current changes."; return; } - const prompt = `${diff}\n\nGenerate a commit message for the above set of changes. First, give a single sentence, no more than 80 characters. Then, after 2 line breaks, give a list of no more than 5 short bullet points, each no more than 40 characters. Output nothing except for the commit message, and don't surround it in quotes.`; - for await (const chunk of llm.streamChat([ - { role: "user", content: prompt }, - ])) { - yield stripImages(chunk.content); + const prompt = `${diff.join("\n")}\n\nGenerate a commit message for the above set of changes. First, give a single sentence, no more than 80 characters. Then, after 2 line breaks, give a list of no more than 5 short bullet points, each no more than 40 characters. Output nothing except for the commit message, and don't surround it in quotes.`; + for await (const chunk of llm.streamChat( + [{ role: "user", content: prompt }], + new AbortController().signal, + )) { + yield renderChatMessage(chunk); } }, }; diff --git a/core/commands/slash/create-readme.ts b/core/commands/slash/create-readme.ts index 263b6312c3..3b65ba75bd 100644 --- a/core/commands/slash/create-readme.ts +++ b/core/commands/slash/create-readme.ts @@ -1,13 +1,13 @@ import { IDE, SlashCommand } from "../.."; import * as fs from "fs/promises"; import * as path from "path"; -import { stripImages } from "../../llm/images"; import ignore from "ignore"; import { defaultIgnoreDir, defaultIgnoreFile, gitIgArrayFromFile, } from "../../indexing/ignore"; +import { renderChatMessage } from "../../util/messageContent"; const LANGUAGE_DEP_MGMT_FILENAMES = [ "package.json", // JavaScript (Node.js) @@ -41,12 +41,13 @@ const CreateReadmeSlashCommand: SlashCommand = { const [workspaceDir] = await ide.getWorkspaceDirs(); const context = await gatherProjectContext(workspaceDir, ide); - const prompt = createProjectFlowPrompt(context, input); + const prompt = createReadmePrompt(context, input); - for await (const chunk of llm.streamChat([ - { role: "user", content: prompt }, - ])) { - yield stripImages(chunk.content); + for await (const chunk of llm.streamChat( + [{ role: "user", content: prompt }], + new AbortController().signal, + )) { + yield renderChatMessage(chunk); } }, }; @@ -112,16 +113,23 @@ async function gatherProjectContext( return context; } -function createProjectFlowPrompt(context: string, input: string): string { +function createReadmePrompt(context: string, input: string): string { return ` - Create a detailed readme.md for the project. - Use the following context about the project structure, READMEs, and dependency files to create a comprehensive overview: - - ${context} - - Please include all key processes. - - ${input} + Please help me generate a detailed and professional README file for my codebase. Below is the context of the codebase, including its purpose, features, setup instructions, and any other relevant information. + + Context: + + - Project Name: [Name of the project] + - Description: [Brief description of what the project does and its purpose] + - Technologies Used: [List of frameworks, libraries, languages, or tools] + - Features: [Highlight the main features of the codebase] + - Setup Instructions: [Steps for users to clone, install dependencies, and run the project] + - Usage Instructions: [Any special instructions for using the project or running tests] + - Contributors: [List of contributors or teams, if any] + - License: [Specify the license type] + - Additional Notes: [Anything extra, like future improvements, acknowledgments, or contact info] + + Format the README with appropriate Markdown syntax for headings, bullet points, and code blocks. Make it clear and user-friendly. `; } diff --git a/core/commands/slash/draftIssue.ts b/core/commands/slash/draftIssue.ts index 25da6b5598..a640dbd25d 100644 --- a/core/commands/slash/draftIssue.ts +++ b/core/commands/slash/draftIssue.ts @@ -1,6 +1,6 @@ import { ChatMessage, SlashCommand } from "../../index.js"; -import { stripImages } from "../../llm/images.js"; import { removeQuotesAndEscapes } from "../../util/index.js"; +import { renderChatMessage } from "../../util/messageContent.js"; const PROMPT = ( input: string, @@ -31,6 +31,7 @@ const DraftIssueCommand: SlashCommand = { } let title = await llm.complete( `Generate a title for the GitHub issue requested in this user input: '${input}'. Use no more than 20 words and output nothing other than the title. Do not surround it with quotes. The title is: `, + new AbortController().signal, { maxTokens: 20 }, ); @@ -43,9 +44,12 @@ const DraftIssueCommand: SlashCommand = { { role: "user", content: PROMPT(input, title) }, ]; - for await (const chunk of llm.streamChat(messages)) { + for await (const chunk of llm.streamChat( + messages, + new AbortController().signal, + )) { body += chunk.content; - yield stripImages(chunk.content); + yield renderChatMessage(chunk); } const url = `${params.repositoryUrl}/issues/new?title=${encodeURIComponent( diff --git a/core/commands/slash/edit.ts b/core/commands/slash/edit.ts deleted file mode 100644 index 5d26b0408d..0000000000 --- a/core/commands/slash/edit.ts +++ /dev/null @@ -1,619 +0,0 @@ -import { ContextItemWithId, ILLM, SlashCommand } from "../../"; -import { - filterCodeBlockLines, - filterEnglishLinesAtEnd, - filterEnglishLinesAtStart, - fixCodeLlamaFirstLineIndentation, - stopAtLines, - streamWithNewLines, -} from "../../autocomplete/streamTransforms/lineStream"; -import { streamLines } from "../../diff/util"; -import { stripImages } from "../../llm/images"; -import { - dedentAndGetCommonWhitespace, - getMarkdownLanguageTagForFile, -} from "../../util/"; -import { - ctxItemToRifWithContents, - type RangeInFileWithContents, -} from "../util"; - -const PROMPT = `Take the file prefix and suffix into account, but only rewrite the code_to_edit as specified in the user_request. The code you write in modified_code_to_edit will replace the code between the code_to_edit tags. Do NOT preface your answer or write anything other than code. The tag should be written to indicate the end of the modified code section. Do not ever use nested tags. - -Example: - - -class Database: - def __init__(self): - self._data = {{}} - - def get(self, key): - return self._data[key] - - - - def set(self, key, value): - self._data[key] = value - - - - def clear_all(): - self._data = {{}} - - -Raise an error if the key already exists. - - - def set(self, key, value): - if key in self._data: - raise KeyError(f"Key {{key}} already exists") - self._data[key] = value - - -Main task: -`; - -export async function getPromptParts( - rif: RangeInFileWithContents, - fullFileContents: string, - model: ILLM, - input: string, - tokenLimit: number | undefined, -) { - const maxTokens = Math.floor(model.contextLength / 2); - - const BUFFER_FOR_FUNCTIONS = 400; - let totalTokens = - model.countTokens(fullFileContents + PROMPT + input) + - BUFFER_FOR_FUNCTIONS + - maxTokens; - - const fullFileContentsList = fullFileContents.split("\n"); - const maxStartLine = rif.range.start.line; - const minEndLine = rif.range.end.line; - let curStartLine = 0; - let curEndLine = fullFileContentsList.length - 1; - - if (totalTokens > model.contextLength) { - while (curEndLine > minEndLine) { - totalTokens -= model.countTokens(fullFileContentsList[curEndLine]); - curEndLine--; - if (totalTokens < model.contextLength) { - break; - } - } - } - - if (totalTokens > model.contextLength) { - while (curStartLine < maxStartLine) { - curStartLine++; - totalTokens -= model.countTokens(fullFileContentsList[curStartLine]); - if (totalTokens < model.contextLength) { - break; - } - } - } - - let filePrefix = fullFileContentsList - .slice(curStartLine, maxStartLine) - .join("\n"); - let fileSuffix = fullFileContentsList - .slice(minEndLine, curEndLine - 1) - .join("\n"); - - if (rif.contents.length > 0) { - let lines = rif.contents.split(/\r?\n/); - let firstLine = lines[0] || null; - while (firstLine && firstLine.trim() === "") { - filePrefix += firstLine; - rif.contents = rif.contents.substring(firstLine.length); - lines = rif.contents.split(/\r?\n/); - firstLine = lines[0] || null; - } - - let lastLine = lines[lines.length - 1] || null; - while (lastLine && lastLine.trim() === "") { - fileSuffix = lastLine + fileSuffix; - rif.contents = rif.contents.substring( - 0, - rif.contents.length - lastLine.length, - ); - lines = rif.contents.split(/\r?\n/); - lastLine = lines[lines.length - 1] || null; - } - - while (rif.contents.startsWith("\n")) { - filePrefix += "\n"; - rif.contents = rif.contents.substring(1); - } - while (rif.contents.endsWith("\n")) { - fileSuffix = `\n${fileSuffix}`; - rif.contents = rif.contents.substring(0, rif.contents.length - 1); - } - } - return { filePrefix, fileSuffix, contents: rif.contents, maxTokens }; -} - -function compilePrompt( - filePrefix: string, - contents: string, - fileSuffix: string, - input: string, -): string { - if (contents.trim() === "") { - // Separate prompt for insertion at the cursor, the other tends to cause it to repeat whole file - return `\ - -${filePrefix} - - - -${fileSuffix} - - -${input} - - -Please output the code to be inserted at the cursor in order to fulfill the user_request. Do NOT preface your answer or write anything other than code. You should not write any tags, just the code. Make sure to correctly indent the code:`; - } - - let prompt = PROMPT; - if (filePrefix.trim() !== "") { - prompt += ` - -${filePrefix} -`; - } - prompt += ` - -${contents} -`; - - if (fileSuffix.trim() !== "") { - prompt += ` - -${fileSuffix} -`; - } - prompt += ` - -${input} - - -`; - - return prompt; -} - -function isEndLine(line: string) { - return ( - line.includes("") || - line.includes("") || - line.includes("[/CODE]") - ); -} - -function lineToBeIgnored(line: string, isFirstLine = false): boolean { - return ( - line.includes("```") || - line.includes("") || - line.includes("") || - line.includes("") || - line.includes("") || - line.includes("") || - line.includes("") || - line.includes("") || - line.includes("") - ); -} - -const EditSlashCommand: SlashCommand = { - name: "edit", - description: "Edit selected code", - run: async function* ({ ide, llm, input, history, contextItems, params }) { - let contextItemToEdit = contextItems.find( - (item: ContextItemWithId) => - item.editing && item.id.providerTitle === "code", - ); - if (!contextItemToEdit) { - contextItemToEdit = contextItems.find( - (item: ContextItemWithId) => item.id.providerTitle === "code", - ); - } - - if (!contextItemToEdit) { - yield "Please highlight the code you want to edit, then press `cmd/ctrl+shift+L` to add it to chat"; - return; - } - - // Strip unecessary parts of the input (the fact that you have to do this is suboptimal, should be refactored away) - let content = history[history.length - 1].content; - if (typeof content !== "string") { - content.forEach((part) => { - if (part.text?.startsWith("/edit")) { - part.text = part.text.replace("/edit", "").trimStart(); - } - }); - } else if (input?.startsWith("/edit")) { - content = input.replace("/edit", "").trimStart(); - } else if (input?.startsWith("/comment")) { - content = input.replace("/comment", "").trimStart(); - } - let userInput = stripImages(content).replace( - `\`\`\`${contextItemToEdit.name}\n${contextItemToEdit.content}\n\`\`\`\n`, - "", - ); - // if the above replace fails to find a match, the code will still be present - // in the userInput. Replace it with input if available. - if (userInput.includes("```") && (input !== "" || !input)) { - userInput = input; - } - - const rif: RangeInFileWithContents = - ctxItemToRifWithContents(contextItemToEdit); - - await ide.saveFile(rif.filepath); - const fullFileContents = await ide.readFile(rif.filepath); - - let { filePrefix, contents, fileSuffix, maxTokens } = await getPromptParts( - rif, - fullFileContents, - llm, - userInput, - params?.tokenLimit, - ); - const [dedentedContents, commonWhitespace] = - dedentAndGetCommonWhitespace(contents); - contents = dedentedContents; - - const prompt = compilePrompt(filePrefix, contents, fileSuffix, userInput); - const fullFileContentsLines = fullFileContents.split("\n"); - const fullPrefixLines = fullFileContentsLines.slice( - 0, - Math.max(0, rif.range.start.line - 1), - ); - const fullSuffixLines = fullFileContentsLines.slice(rif.range.end.line); - - let linesToDisplay: string[] = []; - - async function sendDiffUpdate(lines: string[], final = false) { - const completion = lines.join("\n"); - - // Don't do this at the very end, just show the inserted code - if (final) { - linesToDisplay = []; - } - - // Only recalculate at every new-line, because this is sort of expensive - else if (completion.endsWith("\n")) { - const contentsLines = rif.contents.split("\n"); - let rewrittenLines = 0; - for (const line of lines) { - for (let i = rewrittenLines; i < contentsLines.length; i++) { - if ( - // difflib.SequenceMatcher( - // null, line, contentsLines[i] - // ).ratio() - // > 0.7 - line.trim() === contentsLines[i].trim() && // Temp replacement for difflib (TODO) - contentsLines[i].trim() !== "" - ) { - rewrittenLines = i + 1; - break; - } - } - } - linesToDisplay = contentsLines.slice(rewrittenLines); - } - - const newFileContents = `${fullPrefixLines.join("\n")}\n${completion}\n${ - linesToDisplay.length > 0 ? `${linesToDisplay.join("\n")}\n` : "" - }${fullSuffixLines.join("\n")}`; - - const stepIndex = history.length - 1; - - await ide.showDiff(rif.filepath, newFileContents, stepIndex); - } - - // Important state variables - // ------------------------- - const originalLines = rif.contents === "" ? [] : rif.contents.split("\n"); - // In the actual file, taking into account block offset - let currentLineInFile = rif.range.start.line; - let currentBlockLines: string[] = []; - let originalLinesBelowPreviousBlocks = originalLines; - // The start of the current block in file, taking into account block offset - let currentBlockStart = -1; - let offsetFromBlocks = 0; - - // Don't end the block until you've matched N simultaneous lines - // This helps avoid many tiny blocks - const LINES_TO_MATCH_BEFORE_ENDING_BLOCK = 2; - // If a line has been matched at the end of the block, this is its index within originalLinesBelowPreviousBlocks - // Except we are keeping track of multiple potentialities, so it's a list - // We always check the lines following each of these leads, but if multiple make it out at the end, we use the first one - // This is a tuple of (index_of_last_matched_line, number_of_lines_matched) - let indicesOfLastMatchedLines: [number, number][] = []; - - async function handleGeneratedLine(line: string) { - if (currentBlockLines.length === 0) { - // Set this as the start of the next block - currentBlockStart = - rif.range.start.line + - originalLines.length - - originalLinesBelowPreviousBlocks.length + - offsetFromBlocks; - if ( - originalLinesBelowPreviousBlocks.length > 0 && - line === originalLinesBelowPreviousBlocks[0] - ) { - // Line is equal to the next line in file, move past this line - originalLinesBelowPreviousBlocks = - originalLinesBelowPreviousBlocks.slice(1); - return; - } - } - - // In a block, and have already matched at least one line - // Check if the next line matches, for each of the candidates - const matchesFound: any[] = []; - let firstValidMatch: any = null; - for (const [ - index_of_last_matched_line, - num_lines_matched, - ] of indicesOfLastMatchedLines) { - if ( - index_of_last_matched_line + 1 < - originalLinesBelowPreviousBlocks.length && - line === - originalLinesBelowPreviousBlocks[index_of_last_matched_line + 1] - ) { - matchesFound.push([ - index_of_last_matched_line + 1, - num_lines_matched + 1, - ]); - if ( - firstValidMatch === null && - num_lines_matched + 1 >= LINES_TO_MATCH_BEFORE_ENDING_BLOCK - ) { - firstValidMatch = [ - index_of_last_matched_line + 1, - num_lines_matched + 1, - ]; - } - } - } - indicesOfLastMatchedLines = matchesFound; - - if (firstValidMatch !== null) { - // We've matched the required number of lines, insert suggestion! - - // We added some lines to the block that were matched (including maybe some blank lines) - // So here we will strip all matching lines from the end of currentBlockLines - const linesStripped: string[] = []; - let indexOfLastLineInBlock: number = firstValidMatch[0]; - while ( - currentBlockLines.length > 0 && - currentBlockLines[currentBlockLines.length - 1] === - originalLinesBelowPreviousBlocks[indexOfLastLineInBlock - 1] - ) { - linesStripped.push(currentBlockLines.pop() as string); - indexOfLastLineInBlock -= 1; - } - - // Reset current block / update variables - currentLineInFile += 1; - offsetFromBlocks += currentBlockLines.length; - originalLinesBelowPreviousBlocks = - originalLinesBelowPreviousBlocks.slice(indexOfLastLineInBlock + 1); - currentBlockLines = []; - currentBlockStart = -1; - indicesOfLastMatchedLines = []; - - return; - } - - // Always look for new matching candidates - const newMatches: any[] = []; - for (let i = 0; i < originalLinesBelowPreviousBlocks.length; i++) { - const ogLine = originalLinesBelowPreviousBlocks[i]; - // TODO: It's a bit sus to be disqualifying empty lines. - // What you ideally do is find ALL matches, and then throw them out as you check the following lines - if (ogLine === line) { - // and og_line.trim() !== "": - newMatches.push([i, 1]); - } - } - indicesOfLastMatchedLines = indicesOfLastMatchedLines.concat(newMatches); - - // Make sure they are sorted by index - indicesOfLastMatchedLines = indicesOfLastMatchedLines.sort( - (a, b) => a[0] - b[0], - ); - - currentBlockLines.push(line); - } - - let messages = history; - messages[messages.length - 1] = { role: "user", content: prompt }; - - let linesOfPrefixCopied = 0; - const lines = []; - let unfinishedLine = ""; - let completionLinesCovered = 0; - let repeatingFileSuffix = false; - const lineBelowHighlightedRange = fileSuffix.trim().split("\n")[0]; - - // Use custom templates defined by the model - const template = llm.promptTemplates?.edit; - let generator: AsyncGenerator; - if (template) { - const rendered = llm.renderPromptTemplate( - template, - // typeof template === 'string' ? template : template.prompt, - messages.slice(0, messages.length - 1), - { - codeToEdit: rif.contents, - userInput, - filePrefix: filePrefix, - fileSuffix: fileSuffix, - - // Some built-in templates use these instead of the above - prefix: filePrefix, - suffix: fileSuffix, - - language: getMarkdownLanguageTagForFile(rif.filepath), - systemMessage: llm.systemMessage ?? "", - // "contextItems": (await sdk.getContextItemChatMessages()).map(x => x.content || "").join("\n\n"), - }, - ); - if (typeof rendered === "string") { - messages = [ - { - role: "user", - content: rendered, - }, - ]; - } else { - messages = rendered; - } - - const completion = llm.streamComplete(rendered as string, { - maxTokens: Math.min(maxTokens, Math.floor(llm.contextLength / 2), 4096), - raw: true, - }); - let lineStream = streamLines(completion); - - lineStream = filterEnglishLinesAtStart(lineStream); - - lineStream = filterEnglishLinesAtEnd(filterCodeBlockLines(lineStream)); - lineStream = stopAtLines(lineStream, () => {}); - - generator = streamWithNewLines( - fixCodeLlamaFirstLineIndentation(lineStream), - ); - } else { - async function* gen() { - for await (const chunk of llm.streamChat(messages, { - temperature: 0.5, // TODO - maxTokens: Math.min( - maxTokens, - Math.floor(llm.contextLength / 2), - 4096, - ), - })) { - yield stripImages(chunk.content); - } - } - - generator = gen(); - } - - for await (const chunk of generator) { - // Stop early if it is repeating the fileSuffix or the step was deleted - if (repeatingFileSuffix) { - break; - } - - // Allow stopping breakpoints - yield undefined; - - // Accumulate lines - const chunkLines = chunk.split("\n"); - chunkLines[0] = unfinishedLine + chunkLines[0]; - if (chunk.endsWith("\n")) { - unfinishedLine = ""; - chunkLines.pop(); // because this will be an empty string - } else { - unfinishedLine = chunkLines.pop() ?? ""; - } - - // Deal with newly accumulated lines - for (let i = 0; i < chunkLines.length; i++) { - // Trailing whitespace doesn't matter - chunkLines[i] = chunkLines[i].trimEnd(); - chunkLines[i] = commonWhitespace + chunkLines[i]; - - // Lines that should signify the end of generation - if (isEndLine(chunkLines[i])) { - break; - } - // Lines that should be ignored, like the <> tags - if (lineToBeIgnored(chunkLines[i], completionLinesCovered === 0)) { - continue; // noice - } - // Check if we are currently just copying the prefix - if ( - (linesOfPrefixCopied > 0 || completionLinesCovered === 0) && - linesOfPrefixCopied < filePrefix.split("\n").length && - chunkLines[i] === fullPrefixLines[linesOfPrefixCopied] - ) { - // This is a sketchy way of stopping it from repeating the filePrefix. Is a bug if output happens to have a matching line - linesOfPrefixCopied += 1; - continue; // also nice - } - // Because really short lines might be expected to be repeated, this is only a !heuristic! - // Stop when it starts copying the fileSuffix - if ( - chunkLines[i].trim() === lineBelowHighlightedRange.trim() && - chunkLines[i].trim().length > 4 && - !( - originalLinesBelowPreviousBlocks.length > 0 && - chunkLines[i].trim() === originalLinesBelowPreviousBlocks[0].trim() - ) - ) { - repeatingFileSuffix = true; - break; - } - - lines.push(chunkLines[i]); - completionLinesCovered += 1; - currentLineInFile += 1; - } - - await sendDiffUpdate( - lines.concat([ - unfinishedLine?.startsWith("<") - ? commonWhitespace - : commonWhitespace + unfinishedLine, - ]), - ); - } - - // Add the unfinished line - if ( - unfinishedLine !== "" && - !lineToBeIgnored(unfinishedLine, completionLinesCovered === 0) && - !isEndLine(unfinishedLine) - ) { - unfinishedLine = commonWhitespace + unfinishedLine; - lines.push(unfinishedLine); - await handleGeneratedLine(unfinishedLine); - completionLinesCovered += 1; - currentLineInFile += 1; - } - - await sendDiffUpdate(lines, true); - - if (params?.recap) { - const prompt = `This is the code before editing: -\`\`\` -${contents} -\`\`\` - -This is the code after editing: - -\`\`\` -${lines.join("\n")} -\`\`\` - -Please briefly explain the changes made to the code above. Give no more than 2-3 sentences, and use markdown bullet points:`; - - for await (const update of llm.streamComplete(prompt)) { - yield update; - } - } - }, -}; - -export default EditSlashCommand; diff --git a/core/commands/slash/git-add.ts b/core/commands/slash/git-add.ts index e89105ee11..49db6be701 100644 --- a/core/commands/slash/git-add.ts +++ b/core/commands/slash/git-add.ts @@ -12,9 +12,9 @@ const GitAddAllCommand: SlashCommand = { name: "git:add", description: "Stage all changes for the next commit", run: async function* ({ ide, llm, input }) { - const diff = await ide.getDiff(); + const diff = await ide.getDiff(true); - if (!diff || diff.trim() === "") { + if (!diff) { yield "No changes detected. Make sure you are in a git repository with current changes."; return; } diff --git a/core/commands/slash/git-commit.ts b/core/commands/slash/git-commit.ts index a71ac84c6a..24a460b17e 100644 --- a/core/commands/slash/git-commit.ts +++ b/core/commands/slash/git-commit.ts @@ -12,9 +12,9 @@ const GitCommitCommand: SlashCommand = { name: "git:commit", description: "Commit changes with a custom message", run: async function* ({ ide, llm, input }) { - const diff = await ide.getDiff(); + const diff = await ide.getDiff(false); - if (!diff || diff.trim() === "") { + if (!diff) { yield "No changes detected. Make sure you are in a git repository with current changes."; return; } diff --git a/core/commands/slash/impact-analysis.ts b/core/commands/slash/impact-analysis.ts new file mode 100644 index 0000000000..2072bb7a8b --- /dev/null +++ b/core/commands/slash/impact-analysis.ts @@ -0,0 +1,123 @@ +import { IDE, SlashCommand } from "../.."; +import * as fs from "fs/promises"; +import * as path from "path"; +import ignore from "ignore"; +import { + defaultIgnoreDir, + defaultIgnoreFile, + gitIgArrayFromFile, +} from "../../indexing/ignore"; +import { renderChatMessage } from "../../util/messageContent"; + +const LANGUAGE_DEP_MGMT_FILENAMES = [ + "package.json", // JavaScript (Node.js) + "requirements.txt", // Python + "Gemfile", // Ruby + "pom.xml", // Java (Maven) + "build.gradle", // Java (Gradle) + "composer.json", // PHP + "Cargo.toml", // Rust + "go.mod", // Go + "packages.config", // C# (.NET) + "*.csproj", // C# (.NET Core) + "pubspec.yaml", // Dart + "Project.toml", // Julia + "mix.exs", // Elixir + "rebar.config", // Erlang + "shard.yml", // Crystal + "Package.swift", // Swift + "dependencies.gradle", // Kotlin (when using Gradle) + "Podfile", // Objective-C/Swift (CocoaPods) + "*.cabal", // Haskell + "dub.json", // D +]; + +const MAX_EXPLORE_DEPTH = 2; + +const ImpactAnalysisSlashCommand: SlashCommand = { + name: "impact-analysis", + description: "Generate a real-time impact analysis report", + run: async function* ({ llm, ide }) { + const [workspaceDir] = await ide.getWorkspaceDirs(); + + const context = await gatherProjectContext(workspaceDir, ide); + const prompt = createOnboardingPrompt(context); + + for await (const chunk of llm.streamChat( + [{ role: "user", content: prompt }], + new AbortController().signal, + )) { + yield renderChatMessage(chunk); + } + }, +}; + +async function getEntriesFilteredByIgnore(dir: string, ide: IDE) { + const entries = await fs.readdir(dir, { withFileTypes: true }); + + let ig = ignore().add(defaultIgnoreDir).add(defaultIgnoreFile); + + const gitIgnorePath = path.join(dir, ".gitignore"); + + const hasIgnoreFile = await fs + .access(gitIgnorePath) + .then(() => true) + .catch(() => false); + + if (hasIgnoreFile) { + const gitIgnore = await ide.readFile(gitIgnorePath); + const igPatterns = gitIgArrayFromFile(gitIgnore); + + ig = ig.add(igPatterns); + } + + const filteredEntries = entries.filter((entry) => !ig.ignores(entry.name)); + + return filteredEntries; +} + +async function gatherProjectContext( + workspaceDir: string, + ide: IDE, +): Promise { + let context = ""; + + async function exploreDirectory(dir: string, currentDepth: number = 0) { + if (currentDepth > MAX_EXPLORE_DEPTH) { + return; + } + + const entries = await getEntriesFilteredByIgnore(dir, ide); + + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + const relativePath = path.relative(workspaceDir, fullPath); + + if (entry.isDirectory()) { + context += `\nFolder: ${relativePath}\n`; + await exploreDirectory(fullPath, currentDepth + 1); + } else { + if (entry.name.toLowerCase() === "readme.md") { + const content = await fs.readFile(fullPath, "utf-8"); + context += `README for ${relativePath}:\n${content}\n\n`; + } else if (LANGUAGE_DEP_MGMT_FILENAMES.includes(entry.name)) { + const content = await fs.readFile(fullPath, "utf-8"); + context += `${entry.name} for ${relativePath}:\n${content}\n\n`; + } + } + } + } + + await exploreDirectory(workspaceDir); + + return context; +} + +function createOnboardingPrompt(context: string): string { + return ` + Generate a real-time impact analysis report for the project, highlighting dependencies and their potential impact on the codebase. Provide details on any changes that may affect interconnected components or systems. Additionally, include a 'quick ramp-up' section with documentation links directly associated with each impacted component for a faster understanding of its role and dependencies. + ${context} + `; +} + +export default ImpactAnalysisSlashCommand; diff --git a/core/commands/slash/index.ts b/core/commands/slash/index.ts index 77d5b59ebb..ccdb1eddce 100644 --- a/core/commands/slash/index.ts +++ b/core/commands/slash/index.ts @@ -1,25 +1,21 @@ import GenerateTerminalCommand from "./cmd"; -import CommentSlashCommand from "./comment"; import CommitMessageCommand from "./commit"; import DraftIssueCommand from "./draftIssue"; -import EditSlashCommand from "./edit"; import HttpSlashCommand from "./http"; +import OnboardSlashCommand from "./onboard"; import ReviewMessageCommand from "./review"; import ShareSlashCommand from "./share"; -import StackOverflowSlashCommand from "./stackOverflow"; -import OnboardSlashCommand from "./onboard"; import GitAddAllCommand from "./git-add"; import GitCommitCommand from "./git-commit"; import ProjectFlowSlashCommand from "./project-flow"; import CreateReadmeSlashCommand from "./create-readme"; +import ImpactAnalysisSlashCommand from "./impact-analysis"; +import CreateCodeStatsCommand from "./code-stats"; export default [ DraftIssueCommand, ShareSlashCommand, - StackOverflowSlashCommand, GenerateTerminalCommand, - EditSlashCommand, - CommentSlashCommand, GitAddAllCommand, GitCommitCommand, HttpSlashCommand, @@ -28,4 +24,6 @@ export default [ OnboardSlashCommand, ProjectFlowSlashCommand, CreateReadmeSlashCommand, + ImpactAnalysisSlashCommand, + CreateCodeStatsCommand, ]; diff --git a/core/commands/slash/mcp.ts b/core/commands/slash/mcp.ts new file mode 100644 index 0000000000..a9fa332bbf --- /dev/null +++ b/core/commands/slash/mcp.ts @@ -0,0 +1,46 @@ +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; + +import { ChatMessage, SlashCommand } from "../../index.js"; +import { renderChatMessage } from "../../util/messageContent.js"; +export function constructMcpSlashCommand( + client: Client, + name: string, + description?: string, + args?: string[], +): SlashCommand { + return { + name, + description: description ?? "MCP Prompt", + params: {}, + run: async function* (context) { + const argsObject: { [key: string]: string } = {}; + const userInput = context.input.split(" ").slice(1).join(" "); + if (args) { + args.forEach((arg, i) => { + argsObject[arg] = userInput; + }); + } + + const result = await client.getPrompt({ name, arguments: argsObject }); + + const messages: ChatMessage[] = result.messages.map((msg) => { + if (msg.content.type !== "text") { + throw new Error( + "Continue currently only supports text prompts through MCP", + ); + } + return { + content: msg.content.text, + role: msg.role, + }; + }); + + for await (const chunk of context.llm.streamChat( + messages, + new AbortController().signal, + )) { + yield renderChatMessage(chunk); + } + }, + }; +} diff --git a/core/commands/slash/onboard.ts b/core/commands/slash/onboard.ts index 42f7bc68b3..2d6178ec0d 100644 --- a/core/commands/slash/onboard.ts +++ b/core/commands/slash/onboard.ts @@ -1,13 +1,15 @@ -import { IDE, SlashCommand } from "../.."; import * as fs from "fs/promises"; import * as path from "path"; -import { stripImages } from "../../llm/images"; + import ignore from "ignore"; + +import { IDE, SlashCommand } from "../.."; import { defaultIgnoreDir, defaultIgnoreFile, gitIgArrayFromFile, } from "../../indexing/ignore"; +import { renderChatMessage } from "../../util/messageContent"; const LANGUAGE_DEP_MGMT_FILENAMES = [ "package.json", // JavaScript (Node.js) @@ -43,10 +45,11 @@ const OnboardSlashCommand: SlashCommand = { const context = await gatherProjectContext(workspaceDir, ide); const prompt = createOnboardingPrompt(context); - for await (const chunk of llm.streamChat([ - { role: "user", content: prompt }, - ])) { - yield stripImages(chunk.content); + for await (const chunk of llm.streamChat( + [{ role: "user", content: prompt }], + new AbortController().signal, + )) { + yield renderChatMessage(chunk); } }, }; @@ -70,7 +73,10 @@ async function getEntriesFilteredByIgnore(dir: string, ide: IDE) { ig = ig.add(igPatterns); } - const filteredEntries = entries.filter((entry) => !ig.ignores(entry.name)); + const filteredEntries = entries.filter((entry) => { + const name = entry.isDirectory() ? `${entry.name}/` : entry.name; + return !ig.ignores(name); + }); return filteredEntries; } @@ -114,13 +120,47 @@ async function gatherProjectContext( function createOnboardingPrompt(context: string): string { return ` - I'm a new developer joining the project. + As a helpful AI assistant, your task is to onboard a new developer to this project. Use the following context about the project structure, READMEs, and dependency files to create a comprehensive overview: - ${context} + Please provide an overview of the project with the following guidelines: + + 1. Important Folders + + - Identify the critical folders in the project and explain their purpose. + - Highlight key packages or technologies used within these folders. + - Summarize relevant details from README files or configuration files like package.json. + + 2. Project Architecture + + Here is an example of a valid response: + + ## Important folders + + - Summarize the project's coding standards, including formatting, naming conventions, code structure, and documentation practices. + - Highlight approaches to error handling and testing standards. + + 4. UI Frameworks + + - Explain how UI frameworks are utilized in the project. + - Describe specific use cases and the rationale for using frameworks like MUI, Bootstrap, or Tailwind CSS. + + 5. Environment Configurations + + - Detail how environment configurations are managed, including where they are stored and how they are accessed in the codebase. + + 6. Additional Architectural Insights + + - Provide up to five additional insights about the architecture, such as scalability strategies, CI/CD pipelines, or performance optimizations. + + 7. Unit Testing and Coverage + + - Mention the testing framework(s) used and the approach to ensure sufficient test coverage. + - Highlight strategies for maintaining code quality through testing. + + 8. How to Run the Project - Can you provide a comprehensive overview of the project, including its goals, objectives, and current status? - Please also explain the technologies, frameworks, and tools used, as well as any specific coding conventions or guidelines we should follow. + - Include step-by-step instructions for setting up the project, running it locally, and accessing key functionalities. `; } diff --git a/core/commands/slash/project-flow.ts b/core/commands/slash/project-flow.ts index 5cab39b130..397a35a651 100644 --- a/core/commands/slash/project-flow.ts +++ b/core/commands/slash/project-flow.ts @@ -1,13 +1,13 @@ import { IDE, SlashCommand } from "../.."; import * as fs from "fs/promises"; import * as path from "path"; -import { stripImages } from "../../llm/images"; import ignore from "ignore"; import { defaultIgnoreDir, defaultIgnoreFile, gitIgArrayFromFile, } from "../../indexing/ignore"; +import { renderChatMessage } from "../../util/messageContent"; const LANGUAGE_DEP_MGMT_FILENAMES = [ "package.json", // JavaScript (Node.js) @@ -32,21 +32,30 @@ const LANGUAGE_DEP_MGMT_FILENAMES = [ "dub.json", // D ]; -const MAX_EXPLORE_DEPTH = 2; +const FOLDERS_TO_IGNORE = [ + '.git', + 'node_modules', + '.vscode', + '.idea', + '.github', +] + +const MAX_EXPLORE_DEPTH = 3; const ProjectFlowSlashCommand: SlashCommand = { name: "project-flow", description: "Project Flow chart.", - run: async function* ({ llm, ide }) { + run: async function* ({ llm, ide, input }) { const [workspaceDir] = await ide.getWorkspaceDirs(); const context = await gatherProjectContext(workspaceDir, ide); - const prompt = createProjectFlowPrompt(context); + const prompt = createProjectFlowPrompt(context, input.replace(`/project-flow`, '').trim()); - for await (const chunk of llm.streamChat([ - { role: "user", content: prompt }, - ])) { - yield stripImages(chunk.content); + for await (const chunk of llm.streamChat( + [{ role: "user", content: prompt }], + new AbortController().signal, + )) { + yield renderChatMessage(chunk); } }, }; @@ -93,6 +102,7 @@ async function gatherProjectContext( const relativePath = path.relative(workspaceDir, fullPath); if (entry.isDirectory()) { + if (FOLDERS_TO_IGNORE.includes(relativePath)) return; context += `\nFolder: ${relativePath}\n`; await exploreDirectory(fullPath, currentDepth + 1); } else { @@ -112,14 +122,24 @@ async function gatherProjectContext( return context; } -function createProjectFlowPrompt(context: string): string { +function createProjectFlowPrompt(context: string, input: string): string { return ` - Create a detailed flow diagram for the project. + You are an expert flow diagram creator in text-based flowcharts using ASCII art. + Create a detailed flowchart for the project in a step-by-step manner. Use the following context about the project structure, READMEs, and dependency files to create a comprehensive overview: ${context} - - Please include all key processes, decision points, and data flows. Be sure to label each component clearly and provide a brief description of its function. The diagram should be visually appealing and easy to understand. + ${!!input ? ` + Here is some additional input you may want to use: + + ${input} + + `: ''} + The flowchart should include: + - All key processes, decision points, and data flows. + - Be sure to label each component clearly and provide a brief description of its function. + - The diagram should be visually appealing and easy to understand. + - Please generate or create the flow diagram with logical connectors using ASCII characters such as arrows (-->, |, v) and shapes like [ ] for processes, ( ) for start/end, and < > for decisions. Ensure the flowchart is easy to read and understand, with well-aligned elements. `; } diff --git a/core/commands/slash/review.ts b/core/commands/slash/review.ts index aa4665beaf..b01852e7dc 100644 --- a/core/commands/slash/review.ts +++ b/core/commands/slash/review.ts @@ -1,54 +1,17 @@ -import { ChatMessage, IDE, SlashCommand } from "../../index.js"; -import * as fs from "fs/promises"; -import * as path from "path"; -import { stripImages } from "../../llm/images.js"; -import { - defaultIgnoreDir, - defaultIgnoreFile, - gitIgArrayFromFile -} from "../../indexing/ignore.js"; -import ignore from "ignore"; - -const MAX_EXPLORE_DEPTH = 5; - -const LANGUAGE_DEP_MGMT_FILENAMES = [ - "package.json", // JavaScript (Node.js) - "requirements.txt", // Python - "Gemfile", // Ruby - "pom.xml", // Java (Maven) - "build.gradle", // Java (Gradle) - "composer.json", // PHP - "Cargo.toml", // Rust - "go.mod", // Go - "packages.config", // C# (.NET) - "*.csproj", // C# (.NET Core) - "pubspec.yaml", // Dart - "Project.toml", // Julia - "mix.exs", // Elixir - "rebar.config", // Erlang - "shard.yml", // Crystal - "Package.swift", // Swift - "dependencies.gradle", // Kotlin (when using Gradle) - "Podfile", // Objective-C/Swift (CocoaPods) - "*.cabal", // Haskell - "dub.json", // D -]; +import { ChatMessage, SlashCommand } from "../../index.js"; +import { renderChatMessage } from "../../util/messageContent.js"; const ReviewMessageCommand: SlashCommand = { name: "review", description: "Review code and give feedback", - run: async function* ({ llm, ide, history }) { - const [workspaceDir] = await ide.getWorkspaceDirs(); - const reviewText = getLastUserHistory(history).replace("\\review", ""); - const context = await gatherProjectContext(workspaceDir, ide); - const prompt = createReviewPrompt(context); - - const content = `${prompt} \r\n ${reviewText ? `Please consider this chat history: ${reviewText}` : ""}`; - - for await (const chunk of llm.streamChat([ - { role: "user", content: content }, - ])) { - yield stripImages(chunk.content); + run: async function* ({ llm }) { + const prompt = createReviewPrompt(); + + for await (const chunk of llm.streamChat( + [{ role: "user", content: prompt }], + new AbortController().signal, + )) { + yield renderChatMessage(chunk); } }, }; @@ -78,82 +41,32 @@ function getLastUserHistory(history: ChatMessage[]): string { : ""; } -async function getEntriesFilteredByIgnore(dir: string, ide: IDE) { - const entries = await fs.readdir(dir, { withFileTypes: true }); - - let ig = ignore().add(defaultIgnoreDir).add(defaultIgnoreFile); - - const gitIgnorePath = path.join(dir, ".gitignore"); - - const hasIgnoreFile = await fs - .access(gitIgnorePath) - .then(() => true) - .catch(() => false); - - if (hasIgnoreFile) { - const gitIgnore = await ide.readFile(gitIgnorePath); - const igPatterns = gitIgArrayFromFile(gitIgnore); - - ig = ig.add(igPatterns); - } - - const filteredEntries = entries.filter((entry) => !ig.ignores(entry.name)); - - return filteredEntries; -} - -async function gatherProjectContext( - workspaceDir: string, - ide: IDE, -): Promise { - let context = ""; - - async function exploreDirectory(dir: string, currentDepth: number = 0) { - if (currentDepth > MAX_EXPLORE_DEPTH) { - return; - } - - const entries = await getEntriesFilteredByIgnore(dir, ide); - - for (const entry of entries) { - const fullPath = path.join(dir, entry.name); - const relativePath = path.relative(workspaceDir, fullPath); - - if (entry.isDirectory()) { - context += `\nFolder: ${relativePath}\n`; - await exploreDirectory(fullPath, currentDepth + 1); - } else { - if (entry.name.toLowerCase() === "readme.md") { - const content = await fs.readFile(fullPath, "utf-8"); - context += `README for ${relativePath}:\n${content}\n\n`; - } else if (LANGUAGE_DEP_MGMT_FILENAMES.includes(entry.name)) { - const content = await fs.readFile(fullPath, "utf-8"); - context += `${entry.name} for ${relativePath}:\n${content}\n\n`; - } - } - } - } - - await exploreDirectory(workspaceDir); - - return context; -} - -function createReviewPrompt(context: string): string { +function createReviewPrompt(): string { return ` - Please review the following code. - Use the following context about the project structure, READMEs, and dependency files to create a comprehensive overview: - - ${context} - - Considering these aspects: - - Readability: Is the code easy to understand? - Efficiency: Are there any performance concerns? - Best practices: Does the code follow industry standards and best practices? - Error handling: Is error handling implemented appropriately? - Scalability: Will the code perform well as the system grows? - Documentation: Is the code adequately commented and documented? + You are an expert code reviewer with extensive experience in analyzing and optimizing codebases. + + Review Focus Areas: + + 1. Readability: Assess whether the code is easy to read and understand. Highlight any areas where clarity can be improved. + 2. Efficiency: Identify potential performance bottlenecks or inefficient constructs. Suggest optimizations where applicable. + 3. Best Practices: Evaluate adherence to industry standards and coding conventions. Mention any deviations and their implications. + 4. Error Handling: Check if error handling is implemented effectively. Highlight gaps and propose strategies to handle exceptions gracefully. + 5. Scalability: Analyze whether the codebase is designed to scale efficiently as the system grows in complexity or size. + 6. Documentation: Review the adequacy and quality of inline comments, documentation files, and READMEs. Suggest improvements to enhance maintainability. + 7. Security: Conduct an analysis based on OWASP guidelines and tools like MOB-SF. Identify potential vulnerabilities and recommend mitigation strategies. + + Deliverable: + + Provide a structured and detailed review, including: + + - Specific examples or excerpts from the codebase where applicable. + - Suggestions for improvement for each focus area. + - An overall summary of the codebase's strengths and weaknesses. + + + If you identify any issues or areas of improvement, clearly outline actionable steps for resolving them. + + Kindly ignore irrelevant content. `; } diff --git a/core/commands/slash/share.ts b/core/commands/slash/share.ts index a615ec70da..4b69f4c623 100644 --- a/core/commands/slash/share.ts +++ b/core/commands/slash/share.ts @@ -1,9 +1,10 @@ import * as fs from "node:fs"; import { homedir } from "node:os"; import path from "path"; -import { languageForFilepath } from "../../autocomplete/constructPrompt.js"; + +import { languageForFilepath } from "../../autocomplete/constants/AutocompleteLanguageInfo.js"; import { SlashCommand } from "../../index.js"; -import { stripImages } from "../../llm/images.js"; +import { renderChatMessage } from "../../util/messageContent.js"; // If useful elsewhere, helper funcs should move to core/util/index.ts or similar function getOffsetDatetime(date: Date): Date { @@ -47,7 +48,7 @@ const ShareSlashCommand: SlashCommand = { // message in the chat history, this will omit it for (const msg of history.slice(0, history.length - 1)) { let msgText = msg.content; - msgText = stripImages(msg.content); + msgText = renderChatMessage(msg); if (msg.role === "user" && msgText.search("```") > -1) { msgText = reformatCodeBlocks(msgText); diff --git a/core/commands/slash/stackOverflow.ts b/core/commands/slash/stackOverflow.ts deleted file mode 100644 index 92a34a988f..0000000000 --- a/core/commands/slash/stackOverflow.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { constants } from "../../deploy/constants.js"; -import { ChatMessageRole, FetchFunction, SlashCommand } from "../../index.js"; -import { pruneStringFromBottom } from "../../llm/countTokens.js"; -import { stripImages } from "../../llm/images.js"; - -const PROMPT = ( - input: string, -) => `The above sources are excerpts from related StackOverflow questions. Use them to help answer the below question from our user. Provide links to the sources in markdown whenever possible: - -${input} -`; - -async function getResults(q: string, fetch: FetchFunction): Promise { - const payload = JSON.stringify({ - q: `${q} site:stackoverflow.com`, - }); - - const resp = await fetch(new URL("/search", constants.a), { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: payload, - }); - return await resp.json(); -} - -async function fetchData( - url: string, - fetch: FetchFunction, -): Promise { - const response = await fetch(url, { - headers: { - Accept: "text/html", - }, - }); - const htmlString = await response.text(); - - const parser = new DOMParser(); - const doc = parser.parseFromString(htmlString, "text/html"); - - const h1 = doc.querySelector("h1.fs-headline1"); - const title = h1?.textContent?.trim() ?? "No Title"; - - const bodies = doc.querySelectorAll("div.js-post-body"); - if (bodies.length < 2) { - return undefined; - } - - const question = bodies[0].textContent ?? ""; - const answer = bodies[1].textContent ?? ""; - - return ` - # Question: [${title}](${url}) - -${question} - -# Best Answer - -${answer} - `; -} - -const StackOverflowSlashCommand: SlashCommand = { - name: "so", - description: "Search Stack Overflow", - run: async function* ({ llm, input, addContextItem, history, fetch }) { - const contextLength = llm.contextLength; - - const sources: string[] = []; - const results = await getResults(input, fetch); - const links = results.organic.map((result: any) => result.link); - let totalTokens = llm.countTokens(input) + 200; - - for (const link of links) { - const contents = await fetchData(link, fetch); - if (!contents) { - continue; - } - sources.push(contents); - const newTokens = llm.countTokens(contents); - totalTokens += newTokens; - - let shouldBreak = false; - if (totalTokens > contextLength) { - sources[sources.length - 1] = pruneStringFromBottom( - llm.model, - contextLength - (totalTokens - newTokens), - sources[sources.length - 1], - ); - shouldBreak = true; - } - - if (sources.length >= 3) { - shouldBreak = true; - } - - addContextItem({ - content: sources[sources.length - 1], - description: "StackOverflow Answer", - name: `StackOverflow ${sources.length}`, - id: { - providerTitle: "so", - itemId: links[sources.length - 1], - }, - }); - - if (shouldBreak) { - break; - } - } - - for await (const chunk of llm.streamChat([ - ...history, - ...sources.map((source) => ({ - role: "user" as ChatMessageRole, - content: source, - })), - { role: "user", content: PROMPT(input) }, - ])) { - yield stripImages(chunk.content); - } - }, -}; - -export default StackOverflowSlashCommand; diff --git a/core/commands/util.test.ts b/core/commands/util.test.ts new file mode 100644 index 0000000000..447ec0e600 --- /dev/null +++ b/core/commands/util.test.ts @@ -0,0 +1,149 @@ +import { ctxItemToRifWithContents } from "./util"; +import { ContextItemWithId, RangeInFileWithContents } from "../index"; + +describe("ctxItemToRifWithContents", () => { + it("should parse start and end lines from the item name when format is valid", () => { + const item: ContextItemWithId = { + id: { providerTitle: "testProvider", itemId: "1" }, + name: "myFunction(10-20)", + content: "function content", + description: "test description", + uri: { type: "file", value: "/path/to/file" }, + }; + + const expected: RangeInFileWithContents = { + filepath: "/path/to/file", + range: { + start: { line: 10, character: 0 }, + end: { line: 20, character: 0 }, + }, + contents: "function content", + }; + + const result = ctxItemToRifWithContents(item); + expect(result).toEqual(expected); + }); + + it("should set startLine and endLine to 0 when name format is invalid", () => { + const item: ContextItemWithId = { + id: { providerTitle: "testProvider", itemId: "2" }, + name: "myFunction", + content: "function content", + description: "test description", + uri: { type: "file", value: "/path/to/file" }, + }; + + const expected: RangeInFileWithContents = { + filepath: "/path/to/file", + range: { + start: { line: 0, character: 0 }, + end: { line: 0, character: 0 }, + }, + contents: "function content", + }; + + const result = ctxItemToRifWithContents(item); + expect(result).toEqual(expected); + }); + + it("should handle missing uri by setting filepath to empty string", () => { + const item: ContextItemWithId = { + id: { providerTitle: "testProvider", itemId: "3" }, + name: "myFunction(10-20)", + content: "function content", + description: "test description", + }; + + const expected: RangeInFileWithContents = { + filepath: "", + range: { + start: { line: 10, character: 0 }, + end: { line: 20, character: 0 }, + }, + contents: "function content", + }; + + const result = ctxItemToRifWithContents(item); + expect(result).toEqual(expected); + }); + + it("should handle uri with undefined value by setting filepath to empty string", () => { + const item: ContextItemWithId = { + id: { providerTitle: "testProvider", itemId: "4" }, + name: "myFunction(10-20)", + content: "function content", + description: "test description", + }; + + const expected: RangeInFileWithContents = { + filepath: "", + range: { + start: { line: 10, character: 0 }, + end: { line: 20, character: 0 }, + }, + contents: "function content", + }; + + const result = ctxItemToRifWithContents(item); + expect(result).toEqual(expected); + }); + + it("should handle invalid line numbers gracefully", () => { + const item: ContextItemWithId = { + id: { providerTitle: "testProvider", itemId: "6" }, + name: "myFunction(invalid-lines)", + content: "function content", + description: "test description", + uri: { type: "file", value: "/path/to/file" }, + }; + + const result = ctxItemToRifWithContents(item); + + expect(result.range.start.line).toBeNaN(); + expect(result.range.end.line).toBeNaN(); + }); + + it("should handle missing closing parenthesis in name", () => { + const item: ContextItemWithId = { + id: { providerTitle: "testProvider", itemId: "7" }, + name: "myFunction(10-20", + content: "function content", + description: "test description", + uri: { type: "file", value: "/path/to/file" }, + }; + + const expected: RangeInFileWithContents = { + filepath: "/path/to/file", + range: { + start: { line: 10, character: 0 }, + end: { line: 20, character: 0 }, + }, + contents: "function content", + }; + + const result = ctxItemToRifWithContents(item); + expect(result).toEqual(expected); + }); + + it("should handle name with multiple '-' characters", () => { + const item: ContextItemWithId = { + id: { providerTitle: "testProvider", itemId: "8" }, + name: "myFunction(10-20-30)", + content: "function content", + description: "test description", + uri: { type: "file", value: "/path/to/file" }, + }; + + const expected: RangeInFileWithContents = { + filepath: "/path/to/file", + range: { + start: { line: 10, character: 0 }, + end: { line: 20, character: 0 }, + }, + contents: "function content", + }; + + const result = ctxItemToRifWithContents(item); + expect(result).toEqual(expected); + }); +}); diff --git a/core/commands/util.ts b/core/commands/util.ts index a6be440e9d..74266e70f3 100644 --- a/core/commands/util.ts +++ b/core/commands/util.ts @@ -1,13 +1,4 @@ -import { ContextItemWithId } from "../"; - -export interface RangeInFileWithContents { - filepath: string; - range: { - start: { line: number; character: number }; - end: { line: number; character: number }; - }; - contents: string; -} +import { ContextItemWithId, RangeInFileWithContents } from "../"; export function ctxItemToRifWithContents( item: ContextItemWithId, diff --git a/core/config/ConfigHandler.test.ts b/core/config/ConfigHandler.test.ts index ef4e1814d0..e7d90f492e 100644 --- a/core/config/ConfigHandler.test.ts +++ b/core/config/ConfigHandler.test.ts @@ -1,9 +1,11 @@ import fs from "node:fs"; import path from "node:path"; -import { defaultConfig } from "./default"; + +import { testConfigHandler } from "../test/fixtures"; +import { TEST_DIR } from "../test/testDir"; import { getConfigTsPath } from "../util/paths"; -import { testConfigHandler } from "../test/util/fixtures"; -import { TEST_DIR } from "../test/util/testDir"; + +import { defaultConfig } from "./default"; describe.skip("Test the ConfigHandler and E2E config loading", () => { test("should show only local profile", () => { @@ -28,6 +30,10 @@ describe.skip("Test the ConfigHandler and E2E config loading", () => { fs.writeFileSync(getConfigTsPath(), configTs); await new Promise((resolve) => setTimeout(resolve, 1000)); const config = await testConfigHandler.reloadConfig(); + /** + * @ts-ignore is applied because this test is skipped + */ + // @ts-ignore expect(config.systemMessage).toBe("SYSTEM"); }); @@ -37,6 +43,10 @@ describe.skip("Test the ConfigHandler and E2E config loading", () => { JSON.stringify({ systemMessage: "SYSTEM2" }), ); const config = await testConfigHandler.reloadConfig(); + /** + * @ts-ignore is applied because this test is skipped + */ + // @ts-ignore expect(config.systemMessage).toBe("SYSTEM2"); }); }); diff --git a/core/config/ConfigHandler.ts b/core/config/ConfigHandler.ts index 99f77270fc..3eb55643ff 100644 --- a/core/config/ConfigHandler.ts +++ b/core/config/ConfigHandler.ts @@ -12,97 +12,25 @@ import { } from "../index.js"; import Ollama from "../llm/llms/Ollama.js"; import { GlobalContext } from "../util/GlobalContext.js"; -import { finalToBrowserConfig } from "./load.js"; +import { getConfigJsonPath } from "../util/paths.js"; + +import { ConfigResult } from "./load.js"; import { LOCAL_ONBOARDING_CHAT_MODEL, ONBOARDING_LOCAL_MODEL_TITLE, } from "./onboarding.js"; import ControlPlaneProfileLoader from "./profile/ControlPlaneProfileLoader.js"; -import { IProfileLoader } from "./profile/IProfileLoader.js"; import LocalProfileLoader from "./profile/LocalProfileLoader.js"; +import { + ProfileDescription, + ProfileLifecycleManager, +} from "./ProfileLifecycleManager.js"; -export interface ProfileDescription { - title: string; - id: string; -} - -// Separately manages saving/reloading each profile -class ProfileLifecycleManager { - private savedConfig: ContinueConfig | undefined; - private savedBrowserConfig?: BrowserSerializedContinueConfig; - private pendingConfigPromise?: Promise; - - constructor(private readonly profileLoader: IProfileLoader) {} - - get profileId() { - return this.profileLoader.profileId; - } - - get profileTitle() { - return this.profileLoader.profileTitle; - } - - get profileDescription(): ProfileDescription { - return { - title: this.profileTitle, - id: this.profileId, - }; - } - - clearConfig() { - this.savedConfig = undefined; - this.savedBrowserConfig = undefined; - this.pendingConfigPromise = undefined; - } - - // Clear saved config and reload - async reloadConfig(): Promise { - this.savedConfig = undefined; - this.savedBrowserConfig = undefined; - this.pendingConfigPromise = undefined; - - return await this.profileLoader.doLoadConfig(); - } - - async loadConfig( - additionalContextProviders: IContextProvider[], - ): Promise { - // If we already have a config, return it - if (this.savedConfig) { - return this.savedConfig; - } else if (this.pendingConfigPromise) { - return this.pendingConfigPromise; - } - - // Set pending config promise - this.pendingConfigPromise = new Promise(async (resolve, reject) => { - const newConfig = await this.profileLoader.doLoadConfig(); - - // Add registered context providers - newConfig.contextProviders = (newConfig.contextProviders ?? []).concat( - additionalContextProviders, - ); - - this.savedConfig = newConfig; - resolve(newConfig); - }); +export type { ProfileDescription }; - // Wait for the config promise to resolve - this.savedConfig = await this.pendingConfigPromise; - this.pendingConfigPromise = undefined; - return this.savedConfig; - } +type ConfigUpdateFunction = (payload: ConfigResult) => void; - async getSerializedConfig( - additionalContextProviders: IContextProvider[], - ): Promise { - if (!this.savedBrowserConfig) { - const continueConfig = await this.loadConfig(additionalContextProviders); - this.savedBrowserConfig = finalToBrowserConfig(continueConfig); - } - return this.savedBrowserConfig; - } -} +// Separately manages saving/reloading each profile export class ConfigHandler { private readonly globalContext = new GlobalContext(); @@ -157,52 +85,73 @@ export class ConfigHandler { return this.profiles.filter((p) => p.profileId !== this.selectedProfileId); } + async openConfigProfile(profileId?: string) { + let openProfileId = profileId || this.selectedProfileId; + if (openProfileId === "local") { + await this.ide.openFile(getConfigJsonPath()); + } else { + await this.ide.openUrl( + "https://app.continue.dev/", + // `https://app.continue.dev/workspaces/${openProfileId}/chat`, + ); + } + } + private async fetchControlPlaneProfiles() { // Get the profiles and create their lifecycle managers - this.controlPlaneClient.listWorkspaces().then(async (workspaces) => { - this.profiles = this.profiles.filter( - (profile) => profile.profileId === "local", - ); - workspaces.forEach((workspace) => { - const profileLoader = new ControlPlaneProfileLoader( - workspace.id, - workspace.name, - this.controlPlaneClient, - this.ide, - this.ideSettingsPromise, - this.writeLog, - this.reloadConfig.bind(this), + this.controlPlaneClient + .listWorkspaces() + .then(async (workspaces) => { + this.profiles = this.profiles.filter( + (profile) => profile.profileId === "local", ); - this.profiles.push(new ProfileLifecycleManager(profileLoader)); - }); - - this.notifyProfileListeners( - this.profiles.map((profile) => profile.profileDescription), - ); + workspaces.forEach((workspace) => { + const profileLoader = new ControlPlaneProfileLoader( + workspace.id, + workspace.name, + this.controlPlaneClient, + this.ide, + this.ideSettingsPromise, + this.writeLog, + this.reloadConfig.bind(this), + ); + this.profiles.push(new ProfileLifecycleManager(profileLoader)); + }); - // Check the last selected workspace, and reload if it isn't local - const workspaceId = await this.getWorkspaceId(); - const lastSelectedWorkspaceIds = - this.globalContext.get("lastSelectedProfileForWorkspace") ?? {}; - const selectedWorkspaceId = lastSelectedWorkspaceIds[workspaceId]; - if (selectedWorkspaceId) { - this.selectedProfileId = selectedWorkspaceId; - this.loadConfig(); - } else { - // Otherwise we stick with local profile, and record choice - lastSelectedWorkspaceIds[workspaceId] = this.selectedProfileId; - this.globalContext.update( - "lastSelectedProfileForWorkspace", - lastSelectedWorkspaceIds, + this.notifyProfileListeners( + this.profiles.map((profile) => profile.profileDescription), ); - } - }); + + // Check the last selected workspace, and reload if it isn't local + const workspaceId = await this.getWorkspaceId(); + const lastSelectedWorkspaceIds = + this.globalContext.get("lastSelectedProfileForWorkspace") ?? {}; + const selectedWorkspaceId = lastSelectedWorkspaceIds[workspaceId]; + if (selectedWorkspaceId) { + this.selectedProfileId = selectedWorkspaceId; + await this.loadConfig(); + } else { + // Otherwise we stick with local profile, and record choice + lastSelectedWorkspaceIds[workspaceId] = this.selectedProfileId; + this.globalContext.update( + "lastSelectedProfileForWorkspace", + lastSelectedWorkspaceIds, + ); + } + }) + .catch((e) => { + console.error(e); + }); } async setSelectedProfile(profileId: string) { this.selectedProfileId = profileId; const newConfig = await this.loadConfig(); - this.notifyConfigListeners(newConfig); + this.notifyConfigListeners({ + config: newConfig, + errors: undefined, + configLoadInterrupted: false, + }); const selectedProfiles = this.globalContext.get("lastSelectedProfileForWorkspace") ?? {}; selectedProfiles[await this.getWorkspaceId()] = profileId; @@ -248,24 +197,31 @@ export class ConfigHandler { } } - private notifyConfigListeners(newConfig: ContinueConfig) { + private notifyConfigListeners(result: ConfigResult) { // Notify listeners that config changed for (const listener of this.updateListeners) { - listener(newConfig); + listener(result); } } - private updateListeners: ((newConfig: ContinueConfig) => void)[] = []; - onConfigUpdate(listener: (newConfig: ContinueConfig) => void) { + private updateListeners: ConfigUpdateFunction[] = []; + + onConfigUpdate(listener: ConfigUpdateFunction) { this.updateListeners.push(listener); } async reloadConfig() { // TODO: this isn't right, there are two different senses in which you want to "reload" - const newConfig = await this.currentProfile.reloadConfig(); - this.inactiveProfiles.forEach((profile) => profile.clearConfig()); - this.notifyConfigListeners(newConfig); - return newConfig; + + const { config, errors, configLoadInterrupted } = + await this.currentProfile.reloadConfig(); + + if (config) { + this.inactiveProfiles.forEach((profile) => profile.clearConfig()); + } + + this.notifyConfigListeners({ config, errors, configLoadInterrupted }); + return { config, errors }; } getSerializedConfig(): Promise { diff --git a/core/config/ProfileLifecycleManager.ts b/core/config/ProfileLifecycleManager.ts new file mode 100644 index 0000000000..8d44c8bac9 --- /dev/null +++ b/core/config/ProfileLifecycleManager.ts @@ -0,0 +1,97 @@ +import { + BrowserSerializedContinueConfig, + ContinueConfig, + IContextProvider, +} from "../index.js"; + +import { ConfigResult, finalToBrowserConfig } from "./load.js"; +import { IProfileLoader } from "./profile/IProfileLoader.js"; + +export interface ProfileDescription { + title: string; + id: string; +} + +export class ProfileLifecycleManager { + private savedConfig: ContinueConfig | undefined; + private savedBrowserConfig?: BrowserSerializedContinueConfig; + private pendingConfigPromise?: Promise; + + constructor(private readonly profileLoader: IProfileLoader) {} + + get profileId() { + return this.profileLoader.profileId; + } + + get profileTitle() { + return this.profileLoader.profileTitle; + } + + get profileDescription(): ProfileDescription { + return { + title: this.profileTitle, + id: this.profileId, + }; + } + + clearConfig() { + this.savedConfig = undefined; + this.savedBrowserConfig = undefined; + this.pendingConfigPromise = undefined; + } + + // Clear saved config and reload + async reloadConfig(): Promise> { + this.savedConfig = undefined; + this.savedBrowserConfig = undefined; + this.pendingConfigPromise = undefined; + + return this.profileLoader.doLoadConfig(); + } + + async loadConfig( + additionalContextProviders: IContextProvider[], + ): Promise { + // If we already have a config, return it + if (this.savedConfig) { + return this.savedConfig; + } else if (this.pendingConfigPromise) { + return this.pendingConfigPromise; + } + + // Set pending config promise + this.pendingConfigPromise = new Promise(async (resolve, reject) => { + const { config: newConfig, errors } = + await this.profileLoader.doLoadConfig(); + + if (newConfig) { + // Add registered context providers + newConfig.contextProviders = (newConfig.contextProviders ?? []).concat( + additionalContextProviders, + ); + + this.savedConfig = newConfig; + resolve(newConfig); + } else if (errors) { + reject( + `Error in config.json: ${errors.map((item) => item.message).join(" | ")}`, + ); + } + }); + + // Wait for the config promise to resolve + this.savedConfig = await this.pendingConfigPromise; + this.pendingConfigPromise = undefined; + return this.savedConfig; + } + + async getSerializedConfig( + additionalContextProviders: IContextProvider[], + ): Promise { + if (!this.savedBrowserConfig) { + const continueConfig = await this.loadConfig(additionalContextProviders); + this.savedBrowserConfig = finalToBrowserConfig(continueConfig); + } + return this.savedBrowserConfig; + } +} diff --git a/core/config/default.ts b/core/config/default.ts index a429e0f7e1..bc818c0e1c 100644 --- a/core/config/default.ts +++ b/core/config/default.ts @@ -9,29 +9,23 @@ export const DEFAULT_CHAT_MODEL_CONFIG: ModelDescription[] = [ { "model": "codellama-7b", "provider": "ollama", - "apiBase": "https://apparently-vital-mutt.ngrok-free.app/", - "title": "MB 1" - }, - { - "model": "llama3.2", - "provider": "ollama", - "apiBase": "https://apparently-vital-mutt.ngrok-free.app/", - "title": "MB 0" - }, + "apiBase": "https://pilot.epico.ai/", + "title": "Epico Pilot" + } ]; export const DEFAULT_AUTOCOMPLETE_MODEL_CONFIG: ModelDescription = { "title": "Tab Autocomplete Model", "model": "codellama-7b", "provider": "ollama", - "apiBase": "https://apparently-vital-mutt.ngrok-free.app/" + "apiBase": "https://pilot.epico.ai/" }; export const FREE_TRIAL_MODELS: ModelDescription[] = [ { title: "Claude 3.5 Sonnet (Free Trial)", provider: "free-trial", - model: "claude-3-5-sonnet-20240620", + model: "claude-3-5-sonnet-latest", systemMessage: "You are an expert software developer. You give helpful and concise responses.", }, @@ -65,7 +59,8 @@ export const defaultContextProvidersVsCode: ContextProviderWithParams[] = [ { name: "terminal", params: {} }, { name: "problems", params: {} }, { name: "folder", params: {} }, - { name: "custom-codebase", params: {} }, + { name: "local-codebase", params: {} }, + { name: "remote-codebase", params: {} }, { name: "codebase", params: {} }, ]; @@ -76,14 +71,6 @@ export const defaultContextProvidersJetBrains: ContextProviderWithParams[] = [ ]; export const defaultSlashCommandsVscode: SlashCommandDescription[] = [ - { - name: "edit", - description: "Edit selected code", - }, - { - name: "comment", - description: "Write comments for the selected code", - }, { name: "share", description: "Export the current chat session to markdown", @@ -119,18 +106,18 @@ export const defaultSlashCommandsVscode: SlashCommandDescription[] = [ { name: "create-readme", description: "Create readme file context.", - } -]; - -export const defaultSlashCommandsJetBrains = [ + }, { - name: "edit", - description: "Edit selected code", + name: "impact-analysis", + description: "Generate a real-time impact analysis report", }, { - name: "comment", - description: "Write comments for the selected code", + name: "code-stats", + description: "Generate stats of the codebase.", }, +]; + +export const defaultSlashCommandsJetBrains = [ { name: "share", description: "Export the current chat session to markdown", @@ -144,14 +131,6 @@ export const defaultSlashCommandsJetBrains = [ export const defaultConfig: SerializedContinueConfig = { models: [...DEFAULT_CHAT_MODEL_CONFIG], tabAutocompleteModel: DEFAULT_AUTOCOMPLETE_MODEL_CONFIG, - customCommands: [ - { - name: "test", - prompt: - "{{{ input }}}\n\nWrite a comprehensive set of unit tests for the selected code. It should setup, run tests that check for correctness including important edge cases, and teardown. Ensure that the tests are complete and sophisticated. Give the tests just as chat output, don't edit any file.", - description: "Write unit tests for highlighted code", - }, - ], contextProviders: defaultContextProvidersVsCode, slashCommands: defaultSlashCommandsVscode, }; @@ -159,14 +138,6 @@ export const defaultConfig: SerializedContinueConfig = { export const defaultConfigJetBrains: SerializedContinueConfig = { models: [...DEFAULT_CHAT_MODEL_CONFIG], tabAutocompleteModel: DEFAULT_AUTOCOMPLETE_MODEL_CONFIG, - customCommands: [ - { - name: "test", - prompt: - "{{{ input }}}\n\nWrite a comprehensive set of unit tests for the selected code. It should setup, run tests that check for correctness including important edge cases, and teardown. Ensure that the tests are complete and sophisticated. Give the tests just as chat output, don't edit any file.", - description: "Write unit tests for highlighted code", - }, - ], contextProviders: defaultContextProvidersJetBrains, slashCommands: defaultSlashCommandsJetBrains, }; diff --git a/core/config/getSystemPromptDotFile.ts b/core/config/getSystemPromptDotFile.ts new file mode 100644 index 0000000000..03690dd47e --- /dev/null +++ b/core/config/getSystemPromptDotFile.ts @@ -0,0 +1,27 @@ +import { IDE } from ".."; + +export const SYSTEM_PROMPT_DOT_FILE = ".continuerules"; + +export async function getSystemPromptDotFile(ide: IDE): Promise { + const dirs = await ide.getWorkspaceDirs(); + + let prompts: string[] = []; + const pathSep = await ide.pathSep(); + for (const dir of dirs) { + const dotFile = `${dir}${pathSep}${SYSTEM_PROMPT_DOT_FILE}`; + if (await ide.fileExists(dotFile)) { + try { + const content = await ide.readFile(dotFile); + prompts.push(content); + } catch (e) { + // ignore if file doesn't exist + } + } + } + + if (!prompts.length) { + return null; + } + + return prompts.join("\n\n"); +} diff --git a/core/config/load.ts b/core/config/load.ts index 67982ecad4..eced5d3467 100644 --- a/core/config/load.ts +++ b/core/config/load.ts @@ -1,15 +1,12 @@ -import * as JSONC from "comment-json"; +import { execSync } from "child_process"; import * as fs from "fs"; +import os from "os"; import path from "path"; -import { - slashCommandFromDescription, - slashFromCustomCommand, -} from "../commands/index.js"; -import CustomContextProviderClass from "../context/providers/CustomContextProvider.js"; -import FileContextProvider from "../context/providers/FileContextProvider.js"; -import { contextProviderClassFromName } from "../context/providers/index.js"; -import { AllRerankers } from "../context/rerankers/index.js"; -import { LLMReranker } from "../context/rerankers/llm.js"; + +import { fetchwithRequestOptions } from "@continuedev/fetch"; +import * as JSONC from "comment-json"; +import * as tar from "tar"; + import { BrowserSerializedContinueConfig, Config, @@ -23,45 +20,67 @@ import { IDE, IdeSettings, IdeType, + ILLM, + LLMOptions, ModelDescription, - Reranker, RerankerDescription, SerializedContinueConfig, SlashCommand, -} from "../index.js"; -import TransformersJsEmbeddingsProvider from "../indexing/embeddings/TransformersJsEmbeddingsProvider.js"; -import { allEmbeddingsProviders } from "../indexing/embeddings/index.js"; -import { BaseLLM } from "../llm/index.js"; -import CustomLLMClass from "../llm/llms/CustomLLM.js"; -import FreeTrial from "../llm/llms/FreeTrial.js"; -import { llmFromDescription } from "../llm/llms/index.js"; - -import { execSync } from "child_process"; -import CodebaseContextProvider from "../context/providers/CodebaseContextProvider.js"; -import ContinueProxyContextProvider from "../context/providers/ContinueProxyContextProvider.js"; -import { fetchwithRequestOptions } from "../util/fetchWithOptions.js"; -import { copyOf } from "../util/index.js"; -import mergeJson from "../util/merge.js"; +} from ".."; import { - getConfigJsPath, - getConfigJsPathForRemote, + slashCommandFromDescription, + slashFromCustomCommand, +} from "../commands/index.js"; +import { AllRerankers } from "../context/allRerankers"; +import { MCPManagerSingleton } from "../context/mcp"; +import CodebaseContextProvider from "../context/providers/CodebaseContextProvider"; +import ContinueProxyContextProvider from "../context/providers/ContinueProxyContextProvider"; +import CustomContextProviderClass from "../context/providers/CustomContextProvider"; +import FileContextProvider from "../context/providers/FileContextProvider"; +import { contextProviderClassFromName } from "../context/providers/index"; +import PromptFilesContextProvider from "../context/providers/PromptFilesContextProvider"; +import { allEmbeddingsProviders } from "../indexing/allEmbeddingsProviders"; +import { BaseLLM } from "../llm"; +import { llmFromDescription } from "../llm/llms"; +import CustomLLMClass from "../llm/llms/CustomLLM"; +import FreeTrial from "../llm/llms/FreeTrial"; +import { LLMReranker } from "../llm/llms/llm"; +import TransformersJsEmbeddingsProvider from "../llm/llms/TransformersJsEmbeddingsProvider"; +import { allTools } from "../tools"; +import { copyOf } from "../util"; +import { GlobalContext } from "../util/GlobalContext"; +import mergeJson from "../util/merge"; +import { + DEFAULT_CONFIG_TS_CONTENTS, getConfigJsonPath, getConfigJsonPathForRemote, + getConfigJsPath, + getConfigJsPathForRemote, getConfigTsPath, getContinueDotEnv, + getEsbuildBinaryPath, readAllGlobalPromptFiles, -} from "../util/paths.js"; +} from "../util/paths"; + import { defaultContextProvidersJetBrains, defaultContextProvidersVsCode, defaultSlashCommandsJetBrains, defaultSlashCommandsVscode, -} from "./default.js"; +} from "./default"; +import { getSystemPromptDotFile } from "./getSystemPromptDotFile"; import { DEFAULT_PROMPTS_FOLDER, getPromptFiles, slashCommandFromPromptFile, } from "./promptFile.js"; +import { ConfigValidationError, validateConfig } from "./validation.js"; + +export interface ConfigResult { + config: T | undefined; + errors: ConfigValidationError[] | undefined; + configLoadInterrupted: boolean; +} function resolveSerializedConfig(filepath: string): SerializedContinueConfig { let content = fs.readFileSync(filepath, "utf8"); @@ -97,7 +116,7 @@ function loadSerializedConfig( ideSettings: IdeSettings, ideType: IdeType, overrideConfigJson: SerializedContinueConfig | undefined, -): SerializedContinueConfig { +): ConfigResult { const configPath = getConfigJsonPath(ideType); let config: SerializedContinueConfig = overrideConfigJson!; if (!config) { @@ -108,10 +127,27 @@ function loadSerializedConfig( } } + const errors = validateConfig(config); + + if (errors?.some((error) => error.fatal)) { + return { + errors, + config: undefined, + configLoadInterrupted: true, + }; + } + if (config.allowAnonymousTelemetry === undefined) { config.allowAnonymousTelemetry = true; } + if (config.ui?.getChatTitles === undefined) { + config.ui = { + ...config.ui, + getChatTitles: true, + }; + } + if (ideSettings.remoteConfigServerUrl) { try { const remoteConfigJson = resolveSerializedConfig( @@ -142,7 +178,7 @@ function loadSerializedConfig( ? [...defaultSlashCommandsVscode] : [...defaultSlashCommandsJetBrains]; - return config; + return { config, errors, configLoadInterrupted: false }; } async function serializedToIntermediateConfig( @@ -165,6 +201,7 @@ async function serializedToIntermediateConfig( const promptFolder = initial.experimental?.promptPath; if (loadPromptFiles) { + // v1 prompt files let promptFiles: { path: string; content: string }[] = []; promptFiles = ( await Promise.all( @@ -183,7 +220,10 @@ async function serializedToIntermediateConfig( promptFiles.push(...readAllGlobalPromptFiles()); for (const file of promptFiles) { - slashCommands.push(slashCommandFromPromptFile(file.path, file.content)); + const slashCommand = slashCommandFromPromptFile(file.path, file.content); + if (slashCommand) { + slashCommands.push(slashCommand); + } } } @@ -216,6 +256,7 @@ async function intermediateToFinalConfig( uniqueId: string, writeLog: (log: string) => Promise, workOsAccessToken: string | undefined, + loadPromptFiles: boolean = true, allowFreeTrial: boolean = true, ): Promise { // Auto-detect models @@ -367,6 +408,7 @@ async function intermediateToFinalConfig( const DEFAULT_CONTEXT_PROVIDERS = [ new FileContextProvider({}), new CodebaseContextProvider(codebaseContextParams), + ...(loadPromptFiles ? [new PromptFilesContextProvider({})] : []), ]; const DEFAULT_CONTEXT_PROVIDERS_TITLES = DEFAULT_CONTEXT_PROVIDERS.map( @@ -413,8 +455,12 @@ async function intermediateToFinalConfig( ) { config.embeddingsProvider = new embeddingsProviderClass(); } else { + const llmOptions: LLMOptions = { + model: options.model ?? "UNSPECIFIED", + ...options, + }; config.embeddingsProvider = new embeddingsProviderClass( - options, + llmOptions, (url: string | URL, init: any) => fetchwithRequestOptions(url, init, { ...config.requestOptions, @@ -430,7 +476,7 @@ async function intermediateToFinalConfig( } // Reranker - if (config.reranker && !(config.reranker as Reranker | undefined)?.rerank) { + if (config.reranker && !(config.reranker as ILLM | undefined)?.rerank) { const { name, params } = config.reranker as RerankerDescription; const rerankerClass = AllRerankers[name]; @@ -442,18 +488,58 @@ async function intermediateToFinalConfig( config.reranker = new LLMReranker(llm); } } else if (rerankerClass) { - config.reranker = new rerankerClass(params); + const llmOptions: LLMOptions = { + model: "rerank-2", + ...params, + }; + config.reranker = new rerankerClass(llmOptions); } } - return { + let continueConfig: ContinueConfig = { ...config, contextProviders, models, embeddingsProvider: config.embeddingsProvider as any, tabAutocompleteModels, reranker: config.reranker as any, + tools: allTools, }; + + // Apply MCP if specified + const mcpManager = MCPManagerSingleton.getInstance(); + if (config.experimental?.modelContextProtocolServers) { + config.experimental.modelContextProtocolServers?.forEach( + async (server, index) => { + const mcpId = index.toString(); + const mcpConnection = mcpManager.createConnection(mcpId, server); + if (!mcpConnection) { + return; + } + + const abortController = new AbortController(); + const mcpConnectionTimeout = setTimeout( + () => abortController.abort(), + 2000, + ); + + try { + await mcpConnection.modifyConfig( + continueConfig, + mcpId, + abortController.signal, + ); + } catch (e: any) { + if (e.name !== "AbortError") { + throw e; + } + } + clearTimeout(mcpConnectionTimeout); + }, + ); + } + + return continueConfig; } function finalToBrowserConfig( @@ -486,86 +572,192 @@ function finalToBrowserConfig( disableIndexing: final.disableIndexing, disableSessionTitles: final.disableSessionTitles, userToken: final.userToken, - embeddingsProvider: final.embeddingsProvider?.id, + embeddingsProvider: final.embeddingsProvider?.embeddingId, ui: final.ui, experimental: final.experimental, + docs: final.docs, + tools: final.tools, }; } -function getTarget() { - const os = - { - aix: "linux", - darwin: "darwin", - freebsd: "linux", - linux: "linux", - openbsd: "linux", - sunos: "linux", - win32: "win32", - }[process.platform as string] ?? "linux"; - const arch = { - arm: "arm64", - arm64: "arm64", - ia32: "x64", - loong64: "arm64", - mips: "arm64", - mipsel: "arm64", - ppc: "x64", - ppc64: "x64", - riscv64: "arm64", - s390: "x64", - s390x: "x64", - x64: "x64", - }[process.arch]; - - return `${os}-${arch}`; -} - function escapeSpacesInPath(p: string): string { return p.replace(/ /g, "\\ "); } -async function buildConfigTs() { - if (!fs.existsSync(getConfigTsPath())) { - return undefined; +async function handleEsbuildInstallation(ide: IDE, ideType: IdeType) { + // JetBrains is currently the only IDE that we've reached the plugin size limit and + // therefore need to install esbuild manually to reduce the size + if (ideType !== "jetbrains") { + return; + } + + const globalContext = new GlobalContext(); + if (globalContext.get("hasDismissedConfigTsNoticeJetBrains")) { + return; } + const esbuildPath = getEsbuildBinaryPath(); + + if (fs.existsSync(esbuildPath)) { + return; + } + + console.debug("No esbuild binary detected"); + + const shouldInstall = await promptEsbuildInstallation(ide); + + if (shouldInstall) { + await downloadAndInstallEsbuild(ide); + } +} + +async function promptEsbuildInstallation(ide: IDE): Promise { + const installMsg = "Install esbuild"; + const dismissMsg = "Dismiss"; + + const res = await ide.showToast( + "warning", + "You're using a custom 'config.ts' file, which requires 'esbuild' to be installed. Would you like to install it now?", + dismissMsg, + installMsg, + ); + + if (res === dismissMsg) { + const globalContext = new GlobalContext(); + globalContext.update("hasDismissedConfigTsNoticeJetBrains", true); + return false; + } + + return res === installMsg; +} + +/** + * The download logic is adapted from here: https://esbuild.github.io/getting-started/#download-a-build + */ +async function downloadAndInstallEsbuild(ide: IDE) { + const esbuildPath = getEsbuildBinaryPath(); + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "esbuild-")); + + try { + const target = `${os.platform()}-${os.arch()}`; + const version = "0.19.11"; + const url = `https://registry.npmjs.org/@esbuild/${target}/-/${target}-${version}.tgz`; + const tgzPath = path.join(tempDir, `esbuild-${version}.tgz`); + + console.debug(`Downloading esbuild from: ${url}`); + execSync(`curl -fo "${tgzPath}" "${url}"`); + + console.debug(`Extracting tgz file to: ${tempDir}`); + await tar.x({ + file: tgzPath, + cwd: tempDir, + strip: 2, // Remove the top two levels of directories + }); + + // Ensure the destination directory exists + const destDir = path.dirname(esbuildPath); + if (!fs.existsSync(destDir)) { + fs.mkdirSync(destDir, { recursive: true }); + } + + // Move the file + const extractedBinaryPath = path.join(tempDir, "esbuild"); + fs.renameSync(extractedBinaryPath, esbuildPath); + + // Ensure the binary is executable (not needed on Windows) + if (os.platform() !== "win32") { + fs.chmodSync(esbuildPath, 0o755); + } + + // Clean up + fs.unlinkSync(tgzPath); + fs.rmSync(tempDir, { recursive: true }); + + await ide.showToast( + "info", + `'esbuild' successfully installed to ${esbuildPath}`, + ); + } catch (error) { + console.error("Error downloading or saving esbuild binary:", error); + throw error; + } +} + +async function tryBuildConfigTs() { try { if (process.env.IS_BINARY === "true") { - execSync( - `${escapeSpacesInPath(path.dirname(process.execPath))}/esbuild${ - getTarget().startsWith("win32") ? ".exe" : "" - } ${escapeSpacesInPath( - getConfigTsPath(), - )} --bundle --outfile=${escapeSpacesInPath( - getConfigJsPath(), - )} --platform=node --format=cjs --sourcemap --external:fetch --external:fs --external:path --external:os --external:child_process`, - ); + await buildConfigTsWithBinary(); } else { - // Dynamic import esbuild so potentially disastrous errors can be caught - const esbuild = await import("esbuild"); - - await esbuild.build({ - entryPoints: [getConfigTsPath()], - bundle: true, - platform: "node", - format: "cjs", - outfile: getConfigJsPath(), - external: ["fetch", "fs", "path", "os", "child_process"], - sourcemap: true, - }); + await buildConfigTsWithNodeModule(); } } catch (e) { console.log( `Build error. Please check your ~/.continue/config.ts file: ${e}`, ); - return undefined; } +} + +async function buildConfigTsWithBinary() { + const cmd = [ + escapeSpacesInPath(getEsbuildBinaryPath()), + escapeSpacesInPath(getConfigTsPath()), + "--bundle", + `--outfile=${escapeSpacesInPath(getConfigJsPath())}`, + "--platform=node", + "--format=cjs", + "--sourcemap", + "--external:fetch", + "--external:fs", + "--external:path", + "--external:os", + "--external:child_process", + ].join(" "); + + execSync(cmd); +} - if (!fs.existsSync(getConfigJsPath())) { +async function buildConfigTsWithNodeModule() { + const { build } = await import("esbuild"); + + await build({ + entryPoints: [getConfigTsPath()], + bundle: true, + platform: "node", + format: "cjs", + outfile: getConfigJsPath(), + external: ["fetch", "fs", "path", "os", "child_process"], + sourcemap: true, + }); +} + +function readConfigJs(): string | undefined { + const configJsPath = getConfigJsPath(); + + if (!fs.existsSync(configJsPath)) { return undefined; } - return fs.readFileSync(getConfigJsPath(), "utf8"); + + return fs.readFileSync(configJsPath, "utf8"); +} + +async function buildConfigTsandReadConfigJs(ide: IDE, ideType: IdeType) { + const configTsPath = getConfigTsPath(); + + if (!fs.existsSync(configTsPath)) { + return; + } + + const currentContent = fs.readFileSync(configTsPath, "utf8"); + + // If the user hasn't modified the default config.ts, don't bother building + if (currentContent.trim() === DEFAULT_CONFIG_TS_CONTENTS.trim()) { + return; + } + + await handleEsbuildInstallation(ide, ideType); + await tryBuildConfigTs(); + + return readConfigJs(); } async function loadFullConfigNode( @@ -577,20 +769,33 @@ async function loadFullConfigNode( writeLog: (log: string) => Promise, workOsAccessToken: string | undefined, overrideConfigJson: SerializedContinueConfig | undefined, -): Promise { +): Promise> { // Serialized config - let serialized = loadSerializedConfig( + let { + config: serialized, + errors, + configLoadInterrupted, + } = loadSerializedConfig( workspaceConfigs, ideSettings, ideType, overrideConfigJson, ); + if (!serialized || configLoadInterrupted) { + return { errors, config: undefined, configLoadInterrupted: true }; + } + + const systemPromptDotFile = await getSystemPromptDotFile(ide); + if (systemPromptDotFile) { + serialized.systemMessage = systemPromptDotFile; + } + // Convert serialized to intermediate config let intermediate = await serializedToIntermediateConfig(serialized, ide); // Apply config.ts to modify intermediate config - const configJsContents = await buildConfigTs(); + const configJsContents = await buildConfigTsandReadConfigJs(ide, ideType); if (configJsContents) { try { // Try config.ts first @@ -653,7 +858,7 @@ async function loadFullConfigNode( writeLog, workOsAccessToken, ); - return finalConfig; + return { config: finalConfig, errors, configLoadInterrupted: false }; } export { diff --git a/core/config/onboarding.ts b/core/config/onboarding.ts index 24ab53eb53..bc1d9440e1 100644 --- a/core/config/onboarding.ts +++ b/core/config/onboarding.ts @@ -3,7 +3,7 @@ import { DEFAULT_CHAT_MODEL_CONFIG } from "./default"; export const TRIAL_FIM_MODEL = "codestral-latest"; export const ONBOARDING_LOCAL_MODEL_TITLE = "Ollama"; -export const LOCAL_ONBOARDING_FIM_MODEL = "starcoder2:3b"; +export const LOCAL_ONBOARDING_FIM_MODEL = "qwen2.5-coder:1.5b"; export const LOCAL_ONBOARDING_CHAT_MODEL = "llama3.1:8b"; export const LOCAL_ONBOARDING_CHAT_TITLE = "Llama 3.1 8B"; @@ -34,7 +34,7 @@ export function setupLocalConfig( ...config.models.filter((model) => model.provider !== "free-trial"), ], tabAutocompleteModel: { - title: "Starcoder 3b", + title: "Qwen2.5-Coder 1.5B", provider: "ollama", model: LOCAL_ONBOARDING_FIM_MODEL, }, diff --git a/core/config/profile/ControlPlaneProfileLoader.ts b/core/config/profile/ControlPlaneProfileLoader.ts index 12fe16ec00..448dbdb42d 100644 --- a/core/config/profile/ControlPlaneProfileLoader.ts +++ b/core/config/profile/ControlPlaneProfileLoader.ts @@ -1,4 +1,5 @@ import { ConfigJson } from "@continuedev/config-types"; + import { ControlPlaneClient } from "../../control-plane/client.js"; import { ContinueConfig, @@ -6,8 +7,10 @@ import { IdeSettings, SerializedContinueConfig, } from "../../index.js"; -import { IProfileLoader } from "./IProfileLoader.js"; +import { ConfigResult } from "../load.js"; + import doLoadConfig from "./doLoadConfig.js"; +import { IProfileLoader } from "./IProfileLoader.js"; export default class ControlPlaneProfileLoader implements IProfileLoader { private static RELOAD_INTERVAL = 1000 * 60 * 15; // every 15 minutes @@ -36,7 +39,7 @@ export default class ControlPlaneProfileLoader implements IProfileLoader { }, ControlPlaneProfileLoader.RELOAD_INTERVAL); } - async doLoadConfig(): Promise { + async doLoadConfig(): Promise> { const settings = this.workspaceSettings ?? ((await this.controlPlaneClient.getSettingsForWorkspace( @@ -44,7 +47,7 @@ export default class ControlPlaneProfileLoader implements IProfileLoader { )) as any); const serializedConfig: SerializedContinueConfig = settings; - return doLoadConfig( + const results = await doLoadConfig( this.ide, this.ideSettingsPromise, this.controlPlaneClient, @@ -52,6 +55,11 @@ export default class ControlPlaneProfileLoader implements IProfileLoader { serializedConfig, this.workspaceId, ); + + return { + ...results, + errors: [], // Don't do config validation here, it happens in admin panel + }; } setIsActive(isActive: boolean): void {} diff --git a/core/config/profile/IProfileLoader.ts b/core/config/profile/IProfileLoader.ts index b803e3f68f..2d22a821e1 100644 --- a/core/config/profile/IProfileLoader.ts +++ b/core/config/profile/IProfileLoader.ts @@ -1,11 +1,12 @@ // ProfileHandlers manage the loading of a config, allowing us to abstract over different ways of getting to a ContinueConfig import { ContinueConfig } from "../../index.js"; +import { ConfigResult } from "../load.js"; // After we have the ContinueConfig, the ConfigHandler takes care of everything else (loading models, lifecycle, etc.) export interface IProfileLoader { profileTitle: string; profileId: string; - doLoadConfig(): Promise; + doLoadConfig(): Promise>; setIsActive(isActive: boolean): void; } diff --git a/core/config/profile/LocalProfileLoader.ts b/core/config/profile/LocalProfileLoader.ts index 69cfeb9a5a..0705c87456 100644 --- a/core/config/profile/LocalProfileLoader.ts +++ b/core/config/profile/LocalProfileLoader.ts @@ -1,5 +1,7 @@ import { ControlPlaneClient } from "../../control-plane/client.js"; import { ContinueConfig, IDE, IdeSettings } from "../../index.js"; +import { ConfigResult } from "../load.js"; + import doLoadConfig from "./doLoadConfig.js"; import { IProfileLoader } from "./IProfileLoader.js"; @@ -15,7 +17,7 @@ export default class LocalProfileLoader implements IProfileLoader { private writeLog: (message: string) => Promise, ) {} - async doLoadConfig(): Promise { + async doLoadConfig(): Promise> { return doLoadConfig( this.ide, this.ideSettingsPromise, diff --git a/core/config/profile/doLoadConfig.ts b/core/config/profile/doLoadConfig.ts index e19feed85f..a82dd896de 100644 --- a/core/config/profile/doLoadConfig.ts +++ b/core/config/profile/doLoadConfig.ts @@ -1,22 +1,23 @@ -import { ContinueProxyReranker } from "../../context/rerankers/ContinueProxyReranker.js"; -import { ControlPlaneProxyInfo } from "../../control-plane/analytics/IAnalyticsProvider.js"; -import { - ControlPlaneClient, - DEFAULT_CONTROL_PLANE_PROXY_URL, -} from "../../control-plane/client.js"; -import { TeamAnalytics } from "../../control-plane/TeamAnalytics.js"; +import fs from "fs"; + import { ContinueConfig, ContinueRcJson, IDE, IdeSettings, SerializedContinueConfig, -} from "../../index.js"; -import ContinueProxyEmbeddingsProvider from "../../indexing/embeddings/ContinueProxyEmbeddingsProvider.js"; -import ContinueProxy from "../../llm/llms/stubs/ContinueProxy.js"; -import { Telemetry } from "../../util/posthog.js"; -import { TTS } from "../../util/tts.js"; -import { loadFullConfigNode } from "../load.js"; +} from "../../"; +import { ControlPlaneProxyInfo } from "../../control-plane/analytics/IAnalyticsProvider.js"; +import { ControlPlaneClient } from "../../control-plane/client.js"; +import { controlPlaneEnv } from "../../control-plane/env.js"; +import { TeamAnalytics } from "../../control-plane/TeamAnalytics.js"; +import ContinueProxy from "../../llm/llms/stubs/ContinueProxy"; +import { getConfigYamlPath } from "../../util/paths"; +import { Telemetry } from "../../util/posthog"; +import { TTS } from "../../util/tts"; +import { ConfigResult, loadFullConfigNode } from "../load"; +import { ConfigValidationError } from "../validation"; +import { loadContinueConfigFromYaml } from "../yaml/loadYaml"; export default async function doLoadConfig( ide: IDE, @@ -25,29 +26,54 @@ export default async function doLoadConfig( writeLog: (message: string) => Promise, overrideConfigJson: SerializedContinueConfig | undefined, workspaceId?: string, -) { - let workspaceConfigs: ContinueRcJson[] = []; - try { - workspaceConfigs = await ide.getWorkspaceConfigs(); - } catch (e) { - console.warn("Failed to load workspace configs"); - } - +): Promise> { + const workspaceConfigs = await getWorkspaceConfigs(ide); const ideInfo = await ide.getIdeInfo(); const uniqueId = await ide.getUniqueId(); const ideSettings = await ideSettingsPromise; const workOsAccessToken = await controlPlaneClient.getAccessToken(); - let newConfig = await loadFullConfigNode( - ide, - workspaceConfigs, - ideSettings, - ideInfo.ideType, - uniqueId, - writeLog, - workOsAccessToken, - overrideConfigJson, - ); + const configYamlPath = getConfigYamlPath(ideInfo.ideType); + + let newConfig: ContinueConfig | undefined; + let errors: ConfigValidationError[] | undefined; + let configLoadInterrupted = false; + + if (fs.existsSync(configYamlPath)) { + const result = await loadContinueConfigFromYaml( + ide, + workspaceConfigs.map((c) => JSON.stringify(c)), + ideSettings, + ideInfo.ideType, + uniqueId, + writeLog, + workOsAccessToken, + undefined, + // overrideConfigYaml, TODO + ); + newConfig = result.config; + errors = result.errors; + configLoadInterrupted = result.configLoadInterrupted; + } else { + const result = await loadFullConfigNode( + ide, + workspaceConfigs, + ideSettings, + ideInfo.ideType, + uniqueId, + writeLog, + workOsAccessToken, + overrideConfigJson, + ); + newConfig = result.config; + errors = result.errors; + configLoadInterrupted = result.configLoadInterrupted; + } + + if (configLoadInterrupted || !newConfig) { + return { errors, config: newConfig, configLoadInterrupted: true }; + } + newConfig.allowAnonymousTelemetry = newConfig.allowAnonymousTelemetry && (await ide.isTelemetryEnabled()); @@ -62,9 +88,13 @@ export default async function doLoadConfig( await TTS.setup(); // Set up control plane proxy if configured - let controlPlaneProxyUrl: string = - (newConfig as any).controlPlane?.proxyUrl ?? - DEFAULT_CONTROL_PLANE_PROXY_URL; + const controlPlane = (newConfig as any).controlPlane; + const useOnPremProxy = + controlPlane?.useContinueForTeamsProxy === false && controlPlane?.proxyUrl; + let controlPlaneProxyUrl: string = useOnPremProxy + ? controlPlane?.proxyUrl + : controlPlaneEnv.DEFAULT_CONTROL_PLANE_PROXY_URL; + if (!controlPlaneProxyUrl.endsWith("/")) { controlPlaneProxyUrl += "/"; } @@ -89,7 +119,7 @@ export default async function doLoadConfig( controlPlaneProxyInfo, ); - return newConfig; + return { config: newConfig, errors, configLoadInterrupted: false }; } // Pass ControlPlaneProxyInfo to objects that need it @@ -106,14 +136,30 @@ async function injectControlPlaneProxyInfo( ); if (config.embeddingsProvider?.providerName === "continue-proxy") { - ( - config.embeddingsProvider as ContinueProxyEmbeddingsProvider - ).controlPlaneProxyInfo = info; + (config.embeddingsProvider as ContinueProxy).controlPlaneProxyInfo = info; } - if (config.reranker?.name === "continue-proxy") { - (config.reranker as ContinueProxyReranker).controlPlaneProxyInfo = info; + if (config.reranker?.providerName === "continue-proxy") { + (config.reranker as ContinueProxy).controlPlaneProxyInfo = info; } return config; } + +async function getWorkspaceConfigs(ide: IDE): Promise { + const ideInfo = await ide.getIdeInfo(); + let workspaceConfigs: ContinueRcJson[] = []; + + try { + workspaceConfigs = await ide.getWorkspaceConfigs(); + + // Config is sent over the wire from JB so we need to parse it + if (ideInfo.ideType === "jetbrains") { + workspaceConfigs = (workspaceConfigs as any).map(JSON.parse); + } + } catch (e) { + console.debug("Failed to load workspace configs: ", e); + } + + return workspaceConfigs; +} diff --git a/core/config/promptFile.ts b/core/config/promptFile.ts index 88d57595ac..de17a7120b 100644 --- a/core/config/promptFile.ts +++ b/core/config/promptFile.ts @@ -1,11 +1,21 @@ -import Handlebars from "handlebars"; import path from "path"; + +import Handlebars from "handlebars"; import * as YAML from "yaml"; -import type { IDE, SlashCommand } from ".."; + import { walkDir } from "../indexing/walkDir"; -import { stripImages } from "../llm/images"; -import { renderTemplatedString } from "../promptFiles/renderTemplatedString"; +import { renderTemplatedString } from "../promptFiles/v1/renderTemplatedString"; import { getBasename } from "../util/index"; +import { renderChatMessage } from "../util/messageContent"; + +import type { + ChatMessage, + ContextItem, + ContinueSDK, + IContextProvider, + IDE, + SlashCommand, +} from ".."; export const DEFAULT_PROMPTS_FOLDER = ".prompts"; @@ -89,12 +99,16 @@ export async function createNewPromptFile( export function slashCommandFromPromptFile( path: string, content: string, -): SlashCommand { - const { name, description, systemMessage, prompt } = parsePromptFile( +): SlashCommand | null { + const { name, description, systemMessage, prompt, version } = parsePromptFile( path, content, ); + if (version !== 1) { + return null; + } + return { name, description, @@ -111,8 +125,11 @@ export function slashCommandFromPromptFile( systemMessage, ); - for await (const chunk of context.llm.streamChat(messages)) { - yield stripImages(chunk.content); + for await (const chunk of context.llm.streamChat( + messages, + new AbortController().signal, + )) { + yield renderChatMessage(chunk); } context.llm.systemMessage = originalSystemMessage; @@ -130,6 +147,7 @@ function parsePromptFile(path: string, content: string) { const preamble = YAML.parse(preambleRaw) ?? {}; const name = preamble.name ?? getBasename(path).split(".prompt")[0]; const description = preamble.description ?? name; + const version = preamble.version ?? 2; let systemMessage: string | undefined = undefined; if (prompt.includes("")) { @@ -137,7 +155,7 @@ function parsePromptFile(path: string, content: string) { prompt = prompt.split("")[1].trim(); } - return { name, description, systemMessage, prompt }; + return { name, description, systemMessage, prompt, version }; } function extractUserInput(input: string, commandName: string): string { @@ -147,28 +165,36 @@ function extractUserInput(input: string, commandName: string): string { return input; } -async function renderPrompt(prompt: string, context: any, userInput: string) { +async function renderPrompt( + prompt: string, + context: ContinueSDK, + userInput: string, +) { const helpers = getContextProviderHelpers(context); // A few context providers that don't need to be in config.json to work in .prompt files - const diff = await context.ide.getDiff(); - const currentFilePath = await context.ide.getCurrentFile(); - const currentFile = currentFilePath - ? await context.ide.readFile(currentFilePath) - : undefined; + const diff = await context.ide.getDiff(true); + const currentFile = await context.ide.getCurrentFile(); + const inputData: Record = { + diff: diff.join("\n"), + input: userInput, + }; + if (currentFile) { + inputData.currentFile = currentFile.path; + } return renderTemplatedString( prompt, context.ide.readFile.bind(context.ide), - { diff, currentFile, input: userInput }, + inputData, helpers, ); } function getContextProviderHelpers( - context: any, + context: ContinueSDK, ): Array<[string, Handlebars.HelperDelegate]> | undefined { - return context.config.contextProviders?.map((provider: any) => [ + return context.config.contextProviders?.map((provider: IContextProvider) => [ provider.description.title, async (helperContext: any) => { const items = await provider.getContextItems(helperContext, { @@ -182,16 +208,16 @@ function getContextProviderHelpers( selectedCode: context.selectedCode, }); - items.forEach((item: any) => + items.forEach((item) => context.addContextItem(createContextItem(item, provider)), ); - return items.map((item: any) => item.content).join("\n\n"); + return items.map((item) => item.content).join("\n\n"); }, ]); } -function createContextItem(item: any, provider: any) { +function createContextItem(item: ContextItem, provider: IContextProvider) { return { ...item, id: { @@ -202,7 +228,7 @@ function createContextItem(item: any, provider: any) { } function updateChatHistory( - history: any[], + history: ChatMessage[], commandName: string, renderedPrompt: string, systemMessage?: string, @@ -210,7 +236,8 @@ function updateChatHistory( const messages = [...history]; for (let i = messages.length - 1; i >= 0; i--) { - const { role, content } = messages[i]; + const message = messages[i]; + const { role, content } = message; if (role !== "user") { continue; } @@ -228,7 +255,7 @@ function updateChatHistory( typeof content === "string" && content.startsWith(`/${commandName}`) ) { - messages[i] = { ...messages[i], content: renderedPrompt }; + messages[i] = { ...message, content: renderedPrompt }; break; } } diff --git a/core/config/types.ts b/core/config/types.ts index dafbe588a7..60a2ae8dca 100644 --- a/core/config/types.ts +++ b/core/config/types.ts @@ -1,5 +1,30 @@ const Types = ` declare global { + import Parser from "web-tree-sitter"; + import { GetGhTokenArgs } from "./protocol/ide"; + declare global { + interface Window { + ide?: "vscode"; + windowId: string; + serverUrl: string; + vscMachineId: string; + vscMediaUrl: string; + fullColorTheme?: { + rules?: { + token?: string; + foreground?: string; + }[]; + }; + colorThemeName?: string; + workspacePaths?: string[]; + postIntellijMessage?: ( + messageType: string, + data: any, + messageIde: string, + ) => void; + } + } + export interface ChunkWithoutID { content: string; startLine: number; @@ -7,34 +32,56 @@ declare global { signature?: string; otherMetadata?: { [key: string]: any }; } - + export interface Chunk extends ChunkWithoutID { digest: string; filepath: string; index: number; // Index of the chunk in the document at filepath } - + export interface IndexingProgressUpdate { progress: number; desc: string; shouldClearIndexes?: boolean; - status: "loading" | "indexing" | "done" | "failed" | "paused" | "disabled"; + status: + | "loading" + | "indexing" + | "done" + | "failed" + | "paused" + | "disabled" + | "cancelled"; debugInfo?: string; } - - export type PromptTemplate = - | string - | (( - history: ChatMessage[], - otherData: Record, - ) => string | ChatMessage[]); - + + // This is more or less a V2 of IndexingProgressUpdate for docs etc. + export interface IndexingStatus { + id: string; + type: "docs"; + progress: number; + description: string; + status: "indexing" | "complete" | "paused" | "failed" | "aborted" | "pending"; + embeddingsProviderId: string; + isReindexing?: boolean; + debugInfo?: string; + title: string; + icon?: string; + url?: string; + } + + export type PromptTemplateFunction = ( + history: ChatMessage[], + otherData: Record, + ) => string | ChatMessage[]; + + export type PromptTemplate = string | PromptTemplateFunction; + export interface ILLM extends LLMOptions { - get providerName(): ModelProvider; - + get providerName(): string; + uniqueId: string; model: string; - + title?: string; systemMessage?: string; contextLength: number; @@ -48,48 +95,65 @@ declare global { apiKey?: string; apiBase?: string; cacheBehavior?: CacheBehavior; - - engine?: string; + + deployment?: string; apiVersion?: string; apiType?: string; region?: string; projectId?: string; - - complete(prompt: string, options?: LLMFullCompletionOptions): Promise; - + + // Embedding options + embeddingId: string; + maxEmbeddingChunkSize: number; + maxEmbeddingBatchSize: number; + + complete( + prompt: string, + signal: AbortSignal, + options?: LLMFullCompletionOptions, + ): Promise; + streamComplete( prompt: string, + signal: AbortSignal, options?: LLMFullCompletionOptions, ): AsyncGenerator; - + streamFim( prefix: string, suffix: string, + signal: AbortSignal, options?: LLMFullCompletionOptions, ): AsyncGenerator; - + streamChat( messages: ChatMessage[], + signal: AbortSignal, options?: LLMFullCompletionOptions, ): AsyncGenerator; - + chat( messages: ChatMessage[], + signal: AbortSignal, options?: LLMFullCompletionOptions, ): Promise; - + + embed(chunks: string[]): Promise; + + rerank(query: string, chunks: Chunk[]): Promise; + countTokens(text: string): number; - + supportsImages(): boolean; - + supportsCompletions(): boolean; - + supportsPrefill(): boolean; - + supportsFim(): boolean; - + listModels(): Promise; - + renderPromptTemplate( template: PromptTemplate, history: ChatMessage[], @@ -97,51 +161,54 @@ declare global { canPutWordsInModelsMouth?: boolean, ): string | ChatMessage[]; } - + export type ContextProviderType = "normal" | "query" | "submenu"; - + export interface ContextProviderDescription { - title: string; + title: ContextProviderName; displayTitle: string; description: string; renderInlineAs?: string; type: ContextProviderType; + dependsOnIndexing?: boolean; } - + export type FetchFunction = (url: string | URL, init?: any) => Promise; - + export interface ContextProviderExtras { config: ContinueConfig; fullInput: string; - embeddingsProvider: EmbeddingsProvider; - reranker: Reranker | undefined; + embeddingsProvider: ILLM; + reranker: ILLM | undefined; llm: ILLM; ide: IDE; selectedCode: RangeInFile[]; fetch: FetchFunction; } - + export interface LoadSubmenuItemsArgs { config: ContinueConfig; ide: IDE; fetch: FetchFunction; } - + export interface CustomContextProvider { title: string; displayTitle?: string; description?: string; renderInlineAs?: string; type?: ContextProviderType; + getContextItems( query: string, extras: ContextProviderExtras, ): Promise; + loadSubmenuItems?: ( args: LoadSubmenuItemsArgs, ) => Promise; } - + export interface ContextSubmenuItem { id: string; title: string; @@ -149,7 +216,7 @@ declare global { icon?: string; metadata?: any; } - + export interface SiteIndexingConfig { title: string; startUrl: string; @@ -157,97 +224,151 @@ declare global { maxDepth?: number; faviconUrl?: string; } - + export interface SiteIndexingConfig { startUrl: string; rootUrl?: string; title: string; maxDepth?: number; } - + export interface IContextProvider { get description(): ContextProviderDescription; - + getContextItems( query: string, extras: ContextProviderExtras, ): Promise; - + loadSubmenuItems(args: LoadSubmenuItemsArgs): Promise; } - - export interface PersistedSessionInfo { - history: ChatHistory; + + export interface Checkpoint { + [filepath: string]: string; + } + + export interface Session { + sessionId: string; title: string; workspaceDirectory: string; - sessionId: string; + history: ChatHistoryItem[]; } - - export interface SessionInfo { + + export interface SessionMetadata { sessionId: string; title: string; dateCreated: string; workspaceDirectory: string; } - + export interface RangeInFile { filepath: string; range: Range; } - + export interface Location { filepath: string; position: Position; } - + export interface FileWithContents { filepath: string; contents: string; } - + export interface Range { start: Position; end: Position; } + export interface Position { line: number; character: number; } + export interface FileEdit { filepath: string; range: Range; replacement: string; } - + export interface ContinueError { title: string; message: string; } - + export interface CompletionOptions extends BaseCompletionOptions { model: string; } - - export type ChatMessageRole = "user" | "assistant" | "system"; - + + export type ChatMessageRole = "user" | "assistant" | "system" | "tool"; + export interface MessagePart { type: "text" | "imageUrl"; text?: string; imageUrl?: { url: string }; } - + export type MessageContent = string | MessagePart[]; - - export interface ChatMessage { - role: ChatMessageRole; + + export interface ToolCall { + id: string; + type: "function"; + function: { + name: string; + arguments: string; + }; + } + + export interface ToolCallDelta { + id?: string; + type?: "function"; + function?: { + name?: string; + arguments?: string; + }; + } + + export interface ToolResultChatMessage { + role: "tool"; + content: string; + toolCallId: string; + } + + export interface UserChatMessage { + role: "user"; content: MessageContent; } - + + export interface AssistantChatMessage { + role: "assistant"; + content: MessageContent; + toolCalls?: ToolCallDelta[]; + } + + export interface SystemChatMessage { + role: "system"; + content: string; + } + + export type ChatMessage = + | UserChatMessage + | AssistantChatMessage + | SystemChatMessage + | ToolResultChatMessage; + export interface ContextItemId { providerTitle: string; itemId: string; } - + + export type ContextItemUriTypes = "file" | "url"; + + export interface ContextItemUri { + type: ContextItemUriTypes; + value: string; + } + export interface ContextItem { content: string; name: string; @@ -255,50 +376,74 @@ declare global { editing?: boolean; editable?: boolean; icon?: string; + uri?: ContextItemUri; + hidden?: boolean; } - - export interface ContextItemWithId { - content: string; - name: string; - description: string; + + export interface ContextItemWithId extends ContextItem { id: ContextItemId; - editing?: boolean; - editable?: boolean; - icon?: string; } - + export interface InputModifiers { useCodebase: boolean; noContext: boolean; } - + + export interface SymbolWithRange extends RangeInFile { + name: string; + type: Parser.SyntaxNode["type"]; + content: string; + } + + export type FileSymbolMap = Record; + export interface PromptLog { modelTitle: string; completionOptions: CompletionOptions; prompt: string; completion: string; } - + + type MessageModes = "chat" | "edit"; + + export type ToolStatus = + | "generating" + | "generated" + | "calling" + | "done" + | "canceled"; + + // Will exist only on "assistant" messages with tool calls + interface ToolCallState { + toolCallId: string; + toolCall: ToolCall; + status: ToolStatus; + parsedArgs: any; + output?: ContextItem[]; + } + export interface ChatHistoryItem { message: ChatMessage; + contextItems: ContextItemWithId[]; editorState?: any; modifiers?: InputModifiers; - contextItems: ContextItemWithId[]; promptLogs?: PromptLog[]; + toolCallState?: ToolCallState; + isGatheringContext?: boolean; + checkpoint?: Checkpoint; + isBeforeCheckpoint?: boolean; } - - export type ChatHistory = ChatHistoryItem[]; - - // LLM - + export interface LLMFullCompletionOptions extends BaseCompletionOptions { log?: boolean; - model?: string; } + + export type ToastType = "info" | "error" | "warning"; + export interface LLMOptions { model: string; - + title?: string; uniqueId?: string; systemMessage?: string; @@ -315,35 +460,39 @@ declare global { aiGatewaySlug?: string; apiBase?: string; cacheBehavior?: CacheBehavior; - + useLegacyCompletionsEndpoint?: boolean; - + + // Embedding options + embeddingId?: string; + maxEmbeddingChunkSize?: number; + maxEmbeddingBatchSize?: number; + // Cloudflare options accountId?: string; - + // Azure options - engine?: string; + deployment?: string; apiVersion?: string; apiType?: string; - + // AWS options profile?: string; modelArn?: string; - + // AWS and GCP Options region?: string; - + // GCP Options - projectId?: string; capabilities?: ModelCapability; - - // IBM watsonx options - watsonxUrl?: string; - watsonxCreds?: string; - watsonxProjectId?: string; - watsonxStopToken?: string; - watsonxApiVersion?: string; + + // GCP and Watsonx Options + projectId?: string; + + // IBM watsonx Options + deploymentId?: string; } + type RequireAtLeastOne = Pick< T, Exclude @@ -351,16 +500,18 @@ declare global { { [K in Keys]-?: Required> & Partial>>; }[Keys]; - + export interface CustomLLMWithOptionals { options: LLMOptions; streamCompletion?: ( prompt: string, + signal: AbortSignal, options: CompletionOptions, fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise, ) => AsyncGenerator; streamChat?: ( messages: ChatMessage[], + signal: AbortSignal, options: CompletionOptions, fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise, ) => AsyncGenerator; @@ -368,7 +519,7 @@ declare global { fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise, ) => Promise; } - + /** * The LLM interface requires you to specify either \`streamCompletion\` or \`streamChat\` (or both). */ @@ -376,28 +527,29 @@ declare global { CustomLLMWithOptionals, "streamCompletion" | "streamChat" >; - + // IDE - + export type DiffLineType = "new" | "old" | "same"; - + export interface DiffLine { type: DiffLineType; line: string; } - + export class Problem { filepath: string; range: Range; message: string; } - + export class Thread { name: string; id: number; } - + export type IdeType = "vscode" | "jetbrains"; + export interface IdeInfo { ideType: IdeType; name: string; @@ -405,23 +557,23 @@ declare global { remoteName: string; extensionVersion: string; } - + export interface BranchAndDir { branch: string; directory: string; } - + export interface IndexTag extends BranchAndDir { artifactId: string; } - + export enum FileType { Unkown = 0, File = 1, Directory = 2, SymbolicLink = 64, } - + export interface IdeSettings { remoteConfigServerUrl: string | undefined; remoteConfigSyncPeriod: number; @@ -430,69 +582,119 @@ declare global { pauseCodebaseIndexOnStart: boolean; enableDebugLogs: boolean; } - + export interface IDE { getIdeInfo(): Promise; + getIdeSettings(): Promise; - getDiff(): Promise; + + getDiff(includeUnstaged: boolean): Promise; + + getClipboardContent(): Promise<{ text: string; copiedAt: string }>; + isTelemetryEnabled(): Promise; + getUniqueId(): Promise; + getTerminalContents(): Promise; + getDebugLocals(threadIndex: number): Promise; + getTopLevelCallStackSources( threadIndex: number, stackDepth: number, ): Promise; + getAvailableThreads(): Promise; + listFolders(): Promise; + getWorkspaceDirs(): Promise; + getWorkspaceConfigs(): Promise; + fileExists(filepath: string): Promise; + writeFile(path: string, contents: string): Promise; + showVirtualFile(title: string, contents: string): Promise; + getContinueDir(): Promise; + openFile(path: string): Promise
User Full Name: {{ fullName }}
{{ title }}
{seconds} seconds have passed.
You clicked {count} times
<|fim|>
Description text.
Count: {{ count }}