diff --git a/examples/arithmetics/src/language-server/generated/ast.ts b/examples/arithmetics/src/language-server/generated/ast.ts index 915440201..7b96eae0d 100644 --- a/examples/arithmetics/src/language-server/generated/ast.ts +++ b/examples/arithmetics/src/language-server/generated/ast.ts @@ -17,6 +17,23 @@ export const ArithmeticsTerminals = { export type ArithmeticsTerminalNames = keyof typeof ArithmeticsTerminals; +export type ArithmeticsKeywordNames = + | "%" + | "(" + | ")" + | "*" + | "+" + | "," + | "-" + | "/" + | ":" + | ";" + | "^" + | "def" + | "module"; + +export type ArithmeticsTokenNames = ArithmeticsTerminalNames | ArithmeticsKeywordNames; + export type AbstractDefinition = DeclaredParameter | Definition; export const AbstractDefinition = 'AbstractDefinition'; diff --git a/examples/domainmodel/src/language-server/generated/ast.ts b/examples/domainmodel/src/language-server/generated/ast.ts index b207fec0a..f777fde95 100644 --- a/examples/domainmodel/src/language-server/generated/ast.ts +++ b/examples/domainmodel/src/language-server/generated/ast.ts @@ -16,6 +16,19 @@ export const DomainModelTerminals = { export type DomainModelTerminalNames = keyof typeof DomainModelTerminals; +export type DomainModelKeywordNames = + | "." + | ":" + | "datatype" + | "entity" + | "extends" + | "many" + | "package" + | "{" + | "}"; + +export type DomainModelTokenNames = DomainModelTerminalNames | DomainModelKeywordNames; + export type AbstractElement = PackageDeclaration | Type; export const AbstractElement = 'AbstractElement'; diff --git a/examples/requirements/src/language-server/generated/ast.ts b/examples/requirements/src/language-server/generated/ast.ts index 85d8671e6..bca51919c 100644 --- a/examples/requirements/src/language-server/generated/ast.ts +++ b/examples/requirements/src/language-server/generated/ast.ts @@ -17,6 +17,21 @@ export const RequirementsAndTestsTerminals = { export type RequirementsAndTestsTerminalNames = keyof typeof RequirementsAndTestsTerminals; +export type RequirementsAndTestsKeywordNames = + | "," + | ":" + | "=" + | "applicable" + | "contact" + | "environment" + | "for" + | "req" + | "testFile" + | "tests" + | "tst"; + +export type RequirementsAndTestsTokenNames = RequirementsAndTestsTerminalNames | RequirementsAndTestsKeywordNames; + export interface Contact extends AstNode { readonly $container: RequirementModel | TestModel; readonly $type: 'Contact'; diff --git a/examples/statemachine/src/language-server/generated/ast.ts b/examples/statemachine/src/language-server/generated/ast.ts index a191ba01d..7c0df73e3 100644 --- a/examples/statemachine/src/language-server/generated/ast.ts +++ b/examples/statemachine/src/language-server/generated/ast.ts @@ -16,6 +16,20 @@ export const StatemachineTerminals = { export type StatemachineTerminalNames = keyof typeof StatemachineTerminals; +export type StatemachineKeywordNames = + | "=>" + | "actions" + | "commands" + | "end" + | "events" + | "initialState" + | "state" + | "statemachine" + | "{" + | "}"; + +export type StatemachineTokenNames = StatemachineTerminalNames | StatemachineKeywordNames; + export interface Command extends AstNode { readonly $container: Statemachine; readonly $type: 'Command'; diff --git a/packages/langium-cli/src/generator/ast-generator.ts b/packages/langium-cli/src/generator/ast-generator.ts index c4ebcc517..4b55c325b 100644 --- a/packages/langium-cli/src/generator/ast-generator.ts +++ b/packages/langium-cli/src/generator/ast-generator.ts @@ -4,13 +4,13 @@ * terms of the MIT License, which is available in the project root. ******************************************************************************/ import type { Grammar, LangiumCoreServices } from 'langium'; -import { type Generated, expandToNode, joinToNode, toString } from 'langium/generate'; +import { EOL, type Generated, expandToNode, joinToNode, toString } from 'langium/generate'; import type { AstTypes, Property, PropertyDefaultValue } from 'langium/grammar'; import type { LangiumConfig } from '../package-types.js'; import { AstUtils, MultiMap, GrammarAST } from 'langium'; import { collectAst, collectTypeHierarchy, findReferenceTypes, isAstType, mergeTypesAndInterfaces, escapeQuotes } from 'langium/grammar'; import { generatedHeader } from './node-util.js'; -import { collectTerminalRegexps } from './langium-util.js'; +import { collectKeywords, collectTerminalRegexps } from './langium-util.js'; export function generateAst(services: LangiumCoreServices, grammars: Grammar[], config: LangiumConfig): string { const astTypes = collectAst(grammars, services.shared.workspace.LangiumDocuments); @@ -231,16 +231,26 @@ function groupBySupertypes(astTypes: AstTypes): MultiMap { function generateTerminalConstants(grammars: Grammar[], config: LangiumConfig): Generated { let collection: Record = {}; + const keywordTokens = new Set(); grammars.forEach(grammar => { const terminalConstants = collectTerminalRegexps(grammar); collection = {...collection, ...terminalConstants}; + for (const keyword of collectKeywords(grammar)) { + keywordTokens.add(keyword); + } }); + const keywordStrings = Array.from(keywordTokens).sort().map((keyword) => JSON.stringify(keyword)); + return expandToNode` export const ${config.projectName}Terminals = { ${joinToNode(Object.entries(collection), ([name, regexp]) => `${name}: ${regexp.toString()},`, { appendNewLineIfNotEmpty: true })} }; export type ${config.projectName}TerminalNames = keyof typeof ${config.projectName}Terminals; + + export type ${config.projectName}KeywordNames = ${keywordStrings.length > 0 ? keywordStrings.map(keyword => `${EOL} | ${keyword}`).join('') : 'never'}; + + export type ${config.projectName}TokenNames = ${config.projectName}TerminalNames | ${config.projectName}KeywordNames; `.appendNewLine(); } diff --git a/packages/langium-cli/test/generator/ast-generator.test.ts b/packages/langium-cli/test/generator/ast-generator.test.ts index b15d63346..805b32c51 100644 --- a/packages/langium-cli/test/generator/ast-generator.test.ts +++ b/packages/langium-cli/test/generator/ast-generator.test.ts @@ -461,7 +461,7 @@ function testGeneratedInterface(name: string, grammar: string, expected: string) } function testGeneratedAst(name: string, grammar: string, expected: string): void { - testGenerated(name, grammar, expected, 'export type', 'export type testAstType', 1); + testGenerated(name, grammar, expected, 'export type', 'export type testAstType', 3); } function testTypeMetaData(name: string, grammar: string, expected: string): void { diff --git a/packages/langium/src/languages/generated/ast.ts b/packages/langium/src/languages/generated/ast.ts index 37ffbbf71..41219f074 100644 --- a/packages/langium/src/languages/generated/ast.ts +++ b/packages/langium/src/languages/generated/ast.ts @@ -19,6 +19,60 @@ export const LangiumGrammarTerminals = { export type LangiumGrammarTerminalNames = keyof typeof LangiumGrammarTerminals; +export type LangiumGrammarKeywordNames = + | "!" + | "&" + | "(" + | ")" + | "*" + | "+" + | "+=" + | "," + | "->" + | "." + | ".." + | ":" + | ";" + | "<" + | "=" + | "=>" + | ">" + | "?" + | "?!" + | "? = [begin: TokenName, end: TokenName]; -export interface IndentationTokenBuilderOptions { +export interface IndentationTokenBuilderOptions { /** * The name of the token used to denote indentation in the grammar. * A possible definition in the grammar could look like this: @@ -25,7 +25,7 @@ export interface IndentationTokenBuilderOptions> + ignoreIndentationDelimeters: Array> } export const indentationBuilderDefaultOptions: IndentationTokenBuilderOptions = { @@ -73,15 +73,19 @@ export enum LexingMode { * A token builder that is sensitive to indentation in the input text. * It will generate tokens for indentation and dedentation based on the indentation level. * + * The first generic parameter corresponds to the names of terminal tokens, + * while the second one corresonds to the names of keyword tokens. + * Both parameters are optional and can be imported from `./generated/ast.js`. + * * Inspired by https://github.com/chevrotain/chevrotain/blob/master/examples/lexer/python_indentation/python_indentation.js */ -export class IndentationAwareTokenBuilder extends DefaultTokenBuilder { +export class IndentationAwareTokenBuilder extends DefaultTokenBuilder { /** * The stack in which all the previous matched indentation levels are stored * to understand how deep a the next tokens are nested. */ protected indentationStack: number[] = [0]; - readonly options: IndentationTokenBuilderOptions; + readonly options: IndentationTokenBuilderOptions; /** * The token type to be used for indentation tokens @@ -99,10 +103,10 @@ export class IndentationAwareTokenBuilder ext */ protected whitespaceRegExp = /[ \t]+/y; - constructor(options: Partial>> = indentationBuilderDefaultOptions as IndentationTokenBuilderOptions) { + constructor(options: Partial, NoInfer>> = indentationBuilderDefaultOptions as IndentationTokenBuilderOptions) { super(); this.options = { - ...indentationBuilderDefaultOptions as IndentationTokenBuilderOptions, + ...indentationBuilderDefaultOptions as IndentationTokenBuilderOptions, ...options, };