diff --git a/package.json b/package.json index 532a50e..60f7d0d 100644 --- a/package.json +++ b/package.json @@ -10,8 +10,8 @@ }, "dependencies": { "monaco-editor": "0.21.2", - "vscode-oniguruma": "1.3.0", - "vscode-textmate": "5.1.1" + "vscode-oniguruma": "^1.6.1", + "vscode-textmate": "^6.0.0" }, "devDependencies": { "css-loader": "3.5.3", diff --git a/src/providers.ts b/src/providers.ts index e59ed77..c9b0de8 100644 --- a/src/providers.ts +++ b/src/providers.ts @@ -152,6 +152,8 @@ export class SimpleLanguageInfoProvider { } } +const TOKENIZE_TIMEOUT_MS = 500; + class TokensProviderCache { private scopeNameToGrammar: Map> = new Map(); @@ -172,7 +174,17 @@ class TokensProviderCache { line: string, state: monaco.languages.IState, ): monaco.languages.IEncodedLineTokens { - const tokenizeLineResult2 = grammar.tokenizeLine2(line, state as StackElement); + const tokenizeLineResult2 = grammar.tokenizeLine2( + line, + state as StackElement, + TOKENIZE_TIMEOUT_MS, + ); + + // This is the strategy used by VS Code: + if (tokenizeLineResult2.stoppedEarly) { + return {tokens: tokenizeLineResult2.tokens, endState: state}; + } + const {tokens, ruleStack: endState} = tokenizeLineResult2; return {tokens, endState}; }, diff --git a/yarn.lock b/yarn.lock index 3eafd6f..cce3e81 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4085,15 +4085,15 @@ vm-browserify@^1.0.1: resolved "https://registry.yarnpkg.com/vm-browserify/-/vm-browserify-1.1.2.tgz#78641c488b8e6ca91a75f511e7a3b32a86e5dda0" integrity sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ== -vscode-oniguruma@1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/vscode-oniguruma/-/vscode-oniguruma-1.3.0.tgz#6788a9db2f8b0781243b4eb8c7a1dd25f6c0e2c8" - integrity sha512-m4Br19v6XD4MRbVrgsLNSZgQrBzk1BCMCleL8+GrcoGxKEJJd62zOFcTaoQR3hCrSlLqoxWmJ7Cc0VieVV3iTQ== +vscode-oniguruma@^1.6.1: + version "1.6.2" + resolved "https://registry.yarnpkg.com/vscode-oniguruma/-/vscode-oniguruma-1.6.2.tgz#aeb9771a2f1dbfc9083c8a7fdd9cccaa3f386607" + integrity sha512-KH8+KKov5eS/9WhofZR8M8dMHWN2gTxjMsG4jd04YhpbPR91fUj7rYQ2/XjeHCJWbg7X++ApRIU9NUwM2vTvLA== -vscode-textmate@5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.1.1.tgz#d88dbf271bee7cede455a21bd4894ba5724a4a7e" - integrity sha512-5VHjF+Fglf9d2JI5OyQ7FHutK6/29G0qYyD920K0SWO7uY8JTWbqyKAHEtfB/ZDk2fOe/E23n3wz9fHXKi63yg== +vscode-textmate@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-6.0.0.tgz#a3777197235036814ac9a92451492f2748589210" + integrity sha512-gu73tuZfJgu+mvCSy4UZwd2JXykjK9zAZsfmDeut5dx/1a7FeTk0XwJsSuqQn+cuMCGVbIBfl+s53X4T19DnzQ== wasm-dce@^1.0.0: version "1.0.2"