Improve type-safety of the TokenBuilder's options

examples/arithmetics/src/language-server/generated/ast.ts

@@ -15,6 +15,8 @@ export const ArithmeticsTerminals = {
     SL_COMMENT: /\/\/[^\n\r]*/,
 };
 
+export type ArithmeticsTerminalNames = keyof typeof ArithmeticsTerminals;
+
 export type AbstractDefinition = DeclaredParameter | Definition;
 
 export const AbstractDefinition = 'AbstractDefinition';

examples/domainmodel/src/language-server/generated/ast.ts

@@ -14,6 +14,8 @@ export const DomainModelTerminals = {
     SL_COMMENT: /\/\/[^\n\r]*/,
 };
 
+export type DomainModelTerminalNames = keyof typeof DomainModelTerminals;
+
 export type AbstractElement = PackageDeclaration | Type;
 
 export const AbstractElement = 'AbstractElement';

examples/requirements/src/language-server/generated/ast.ts

@@ -15,6 +15,8 @@ export const RequirementsAndTestsTerminals = {
     SL_COMMENT: /\/\/[^\n\r]*/,
 };
 
+export type RequirementsAndTestsTerminalNames = keyof typeof RequirementsAndTestsTerminals;
+
 export interface Contact extends AstNode {
     readonly $container: RequirementModel | TestModel;
     readonly $type: 'Contact';

examples/statemachine/src/language-server/generated/ast.ts

@@ -14,6 +14,8 @@ export const StatemachineTerminals = {
     SL_COMMENT: /\/\/[^\n\r]*/,
 };
 
+export type StatemachineTerminalNames = keyof typeof StatemachineTerminals;
+
 export interface Command extends AstNode {
     readonly $container: Statemachine;
     readonly $type: 'Command';

package.json

@@ -31,16 +31,16 @@
     "@vitest/coverage-v8": "~1.0.0",
     "@vitest/ui": "~1.5.0",
     "concurrently": "~8.2.1",
+    "editorconfig": "~2.0.0",
     "esbuild": "~0.19.2",
     "eslint": "~8.56.0",
     "eslint-plugin-header": "~3.1.1",
-    "editorconfig": "~2.0.0",
     "shx": "~0.3.4",
-    "typescript": "~5.1.6",
+    "typescript": "~5.4.5",
     "vitest": "~1.5.0"
   },
   "overrides": {
-   "@types/node": "~16.18.41"
+    "@types/node": "~16.18.41"
   },
   "volta": {
     "node": "18.19.1",

packages/langium-cli/src/generator/ast-generator.ts

@@ -240,6 +240,7 @@ function generateTerminalConstants(grammars: Grammar[], config: LangiumConfig):
         export const ${config.projectName}Terminals = {
             ${joinToNode(Object.entries(collection), ([name, regexp]) => `${name}: ${regexp.toString()},`, { appendNewLineIfNotEmpty: true })}
         };
+
+        export type ${config.projectName}TerminalNames = keyof typeof ${config.projectName}Terminals;
     `.appendNewLine();
 }
-

packages/langium-cli/test/generator/ast-generator.test.ts

@@ -461,14 +461,14 @@ function testGeneratedInterface(name: string, grammar: string, expected: string)
 }
 
 function testGeneratedAst(name: string, grammar: string, expected: string): void {
-    testGenerated(name, grammar, expected, 'export type', 'export type testAstType');
+    testGenerated(name, grammar, expected, 'export type', 'export type testAstType', 1);
 }
 
 function testTypeMetaData(name: string, grammar: string, expected: string): void {
     testGenerated(name, grammar, expected, 'getTypeMetaData', 'export const reflection');
 }
 
-function testGenerated(name: string, grammar: string, expected: string, start: string, end: string): void {
+function testGenerated(name: string, grammar: string, expected: string, start: string, end: string, startCount = 0): void {
     test(name, async () => {
         const result = (await parse(grammar)).parseResult;
         const config: LangiumConfig = {
@@ -478,7 +478,11 @@ function testGenerated(name: string, grammar: string, expected: string, start: s
         };
         const expectedPart = normalizeEOL(expected).trim();
         const typesFileContent = generateAst(services.grammar, [result.value], config);
-        const relevantPart = typesFileContent.substring(typesFileContent.indexOf(start), typesFileContent.indexOf(end)).trim();
+        let startIndex = typesFileContent.indexOf(start);
+        for (let i = 0; i < startCount; i++) {
+            startIndex = typesFileContent.indexOf(start, startIndex + start.length);
+        }
+        const relevantPart = typesFileContent.substring(startIndex, typesFileContent.indexOf(end)).trim();
         expect(relevantPart).toEqual(expectedPart);
     });
 }

packages/langium/src/languages/generated/ast.ts

@@ -17,6 +17,8 @@ export const LangiumGrammarTerminals = {
     SL_COMMENT: /\/\/[^\n\r]*/,
 };
 
+export type LangiumGrammarTerminalNames = keyof typeof LangiumGrammarTerminals;
+
 export type AbstractRule = ParserRule | TerminalRule;
 
 export const AbstractRule = 'AbstractRule';

packages/langium/src/parser/indentation-aware.ts

@@ -13,7 +13,7 @@ import { createToken, createTokenInstance, Lexer } from 'chevrotain';
 import { DefaultTokenBuilder } from './token-builder.js';
 import { DefaultLexer, isTokenTypeArray } from './lexer.js';
 
-export interface IndentationTokenBuilderOptions {
+export interface IndentationTokenBuilderOptions<TokenName extends string = string> {
     /**
      * The name of the token used to denote indentation in the grammar.
      * A possible definition in the grammar could look like this:
@@ -23,7 +23,7 @@ export interface IndentationTokenBuilderOptions {
      *
      * @default 'INDENT'
      */
-    indentTokenName: string;
+    indentTokenName: TokenName;
     /**
      * The name of the token used to denote deindentation in the grammar.
      * A possible definition in the grammar could look like this:
@@ -33,7 +33,7 @@ export interface IndentationTokenBuilderOptions {
      *
      * @default 'DEDENT'
      */
-    dedentTokenName: string;
+    dedentTokenName: TokenName;
     /**
      * The name of the token used to denote whitespace other than indentation and newlines in the grammar.
      * A possible definition in the grammar could look like this:
@@ -43,7 +43,7 @@ export interface IndentationTokenBuilderOptions {
      *
      * @default 'WS'
      */
-    whitespaceTokenName: string;
+    whitespaceTokenName: TokenName;
 }
 
 export const indentationBuilderDefaultOptions: IndentationTokenBuilderOptions = {
@@ -58,13 +58,13 @@ export const indentationBuilderDefaultOptions: IndentationTokenBuilderOptions =
  *
  * Inspired by https://github.com/chevrotain/chevrotain/blob/master/examples/lexer/python_indentation/python_indentation.js
  */
-export class IndentationAwareTokenBuilder extends DefaultTokenBuilder {
+export class IndentationAwareTokenBuilder<Terminals extends string = string> extends DefaultTokenBuilder {
     /**
      * The stack in which all the previous matched indentation levels are stored
      * to understand how deep a the next tokens are nested.
      */
     protected indentationStack: number[] = [0];
-    readonly options: IndentationTokenBuilderOptions;
+    readonly options: IndentationTokenBuilderOptions<Terminals>;
 
     /**
      * The token type to be used for indentation tokens
@@ -82,10 +82,10 @@ export class IndentationAwareTokenBuilder extends DefaultTokenBuilder {
      */
     protected whitespaceRegExp = /[ \t]+/y;
 
-    constructor(options: Partial<IndentationTokenBuilderOptions> = indentationBuilderDefaultOptions) {
+    constructor(options: Partial<IndentationTokenBuilderOptions<NoInfer<Terminals>>> = indentationBuilderDefaultOptions as IndentationTokenBuilderOptions<Terminals>) {
         super();
         this.options = {
-            ...indentationBuilderDefaultOptions,
+            ...indentationBuilderDefaultOptions as IndentationTokenBuilderOptions<Terminals>,
             ...options,
         };