From 644660a14148787f428ab440e53ea7821c7c9ff7 Mon Sep 17 00:00:00 2001
From: Mark Sujew <mark.sujew@typefox.io>
Date: Mon, 18 Nov 2024 11:27:38 +0100
Subject: [PATCH] Add lexer error message provider (#1716)

---
 packages/langium/src/default-module.ts |  5 +++--
 packages/langium/src/parser/lexer.ts   | 24 +++++++++++++++++++-----
 packages/langium/src/services.ts       |  3 ++-
 3 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/packages/langium/src/default-module.ts b/packages/langium/src/default-module.ts
index d3aab8fc4..b96c1a097 100644
--- a/packages/langium/src/default-module.ts
+++ b/packages/langium/src/default-module.ts
@@ -28,7 +28,7 @@ import { DefaultDocumentBuilder } from './workspace/document-builder.js';
 import { DefaultLangiumDocumentFactory, DefaultLangiumDocuments } from './workspace/documents.js';
 import { DefaultIndexManager } from './workspace/index-manager.js';
 import { DefaultWorkspaceManager } from './workspace/workspace-manager.js';
-import { DefaultLexer } from './parser/lexer.js';
+import { DefaultLexer, DefaultLexerErrorMessageProvider } from './parser/lexer.js';
 import { JSDocDocumentationProvider } from './documentation/documentation-provider.js';
 import { DefaultCommentProvider } from './documentation/comment-provider.js';
 import { LangiumParserErrorMessageProvider } from './parser/langium-parser.js';
@@ -61,7 +61,8 @@ export function createDefaultCoreModule(context: DefaultCoreModuleContext): Modu
             ValueConverter: () => new DefaultValueConverter(),
             TokenBuilder: () => new DefaultTokenBuilder(),
             Lexer: (services) => new DefaultLexer(services),
-            ParserErrorMessageProvider: () => new LangiumParserErrorMessageProvider()
+            ParserErrorMessageProvider: () => new LangiumParserErrorMessageProvider(),
+            LexerErrorMessageProvider: () => new DefaultLexerErrorMessageProvider()
         },
         workspace: {
             AstNodeLocator: () => new DefaultAstNodeLocator(),
diff --git a/packages/langium/src/parser/lexer.ts b/packages/langium/src/parser/lexer.ts
index b387ebcad..bcabafaf3 100644
--- a/packages/langium/src/parser/lexer.ts
+++ b/packages/langium/src/parser/lexer.ts
@@ -4,11 +4,22 @@
  * terms of the MIT License, which is available in the project root.
  ******************************************************************************/
 
-import type { ILexingError, IMultiModeLexerDefinition, IToken, TokenType, TokenTypeDictionary, TokenVocabulary } from 'chevrotain';
+import type { ILexerErrorMessageProvider, ILexingError, IMultiModeLexerDefinition, IToken, TokenType, TokenTypeDictionary, TokenVocabulary } from 'chevrotain';
 import type { LangiumCoreServices } from '../services.js';
-import { Lexer as ChevrotainLexer } from 'chevrotain';
+import { Lexer as ChevrotainLexer, defaultLexerErrorProvider } from 'chevrotain';
 import type { LexingReport, TokenBuilder } from './token-builder.js';
 
+export class DefaultLexerErrorMessageProvider implements ILexerErrorMessageProvider {
+
+    buildUnexpectedCharactersMessage(fullText: string, startOffset: number, length: number, line?: number, column?: number): string {
+        return defaultLexerErrorProvider.buildUnexpectedCharactersMessage(fullText, startOffset, length, line, column);
+    }
+
+    buildUnableToPopLexerModeMessage(token: IToken): string {
+        return defaultLexerErrorProvider.buildUnableToPopLexerModeMessage(token);
+    }
+}
+
 export interface LexerResult {
     /**
      * A list of all tokens that were lexed from the input.
@@ -40,11 +51,13 @@ export interface Lexer {
 
 export class DefaultLexer implements Lexer {
 
-    protected chevrotainLexer: ChevrotainLexer;
-    protected tokenBuilder: TokenBuilder;
+    protected readonly tokenBuilder: TokenBuilder;
+    protected readonly errorMessageProvider: ILexerErrorMessageProvider;
     protected tokenTypes: TokenTypeDictionary;
+    protected chevrotainLexer: ChevrotainLexer;
 
     constructor(services: LangiumCoreServices) {
+        this.errorMessageProvider = services.parser.LexerErrorMessageProvider;
         this.tokenBuilder = services.parser.TokenBuilder;
         const tokens = this.tokenBuilder.buildTokens(services.Grammar, {
             caseInsensitive: services.LanguageMetaData.caseInsensitive
@@ -54,7 +67,8 @@ export class DefaultLexer implements Lexer {
         const production = services.LanguageMetaData.mode === 'production';
         this.chevrotainLexer = new ChevrotainLexer(lexerTokens, {
             positionTracking: 'full',
-            skipValidations: production
+            skipValidations: production,
+            errorMessageProvider: this.errorMessageProvider
         });
     }
 
diff --git a/packages/langium/src/services.ts b/packages/langium/src/services.ts
index 06627388f..a4760c42d 100644
--- a/packages/langium/src/services.ts
+++ b/packages/langium/src/services.ts
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 // Ensure that all imports are erased at runtime to avoid circular dependencies.
-import type { IParserErrorMessageProvider } from 'chevrotain';
+import type { IParserErrorMessageProvider, ILexerErrorMessageProvider } from 'chevrotain';
 import type { CommentProvider } from './documentation/comment-provider.js';
 import type { DocumentationProvider } from './documentation/documentation-provider.js';
 import type { Grammar } from './languages/generated/ast.js';
@@ -60,6 +60,7 @@ export type LangiumDefaultCoreServices = {
         readonly ValueConverter: ValueConverter
         readonly LangiumParser: LangiumParser
         readonly ParserErrorMessageProvider: IParserErrorMessageProvider
+        readonly LexerErrorMessageProvider: ILexerErrorMessageProvider
         readonly CompletionParser: LangiumCompletionParser
         readonly TokenBuilder: TokenBuilder
         readonly Lexer: Lexer