From d1f330bdb422c6e9b4f3fab4f4580ee7919f22f2 Mon Sep 17 00:00:00 2001 From: Abdelrahman Abounegm Date: Sun, 25 Aug 2024 22:12:05 +0300 Subject: [PATCH 1/4] Add keyword tokens to the terminals type --- packages/langium-cli/src/generator/ast-generator.ts | 11 ++++++++++- .../langium-cli/test/generator/ast-generator.test.ts | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/langium-cli/src/generator/ast-generator.ts b/packages/langium-cli/src/generator/ast-generator.ts index c4ebcc517..ba93abe93 100644 --- a/packages/langium-cli/src/generator/ast-generator.ts +++ b/packages/langium-cli/src/generator/ast-generator.ts @@ -10,7 +10,7 @@ import type { LangiumConfig } from '../package-types.js'; import { AstUtils, MultiMap, GrammarAST } from 'langium'; import { collectAst, collectTypeHierarchy, findReferenceTypes, isAstType, mergeTypesAndInterfaces, escapeQuotes } from 'langium/grammar'; import { generatedHeader } from './node-util.js'; -import { collectTerminalRegexps } from './langium-util.js'; +import { collectKeywords, collectTerminalRegexps } from './langium-util.js'; export function generateAst(services: LangiumCoreServices, grammars: Grammar[], config: LangiumConfig): string { const astTypes = collectAst(grammars, services.shared.workspace.LangiumDocuments); @@ -231,16 +231,25 @@ function groupBySupertypes(astTypes: AstTypes): MultiMap { function generateTerminalConstants(grammars: Grammar[], config: LangiumConfig): Generated { let collection: Record = {}; + const keywordTokens: string[] = []; grammars.forEach(grammar => { const terminalConstants = collectTerminalRegexps(grammar); collection = {...collection, ...terminalConstants}; + const keywords = collectKeywords(grammar); + keywordTokens.push(...keywords); }); + const keywordStrings = keywordTokens.map((keyword) => JSON.stringify(keyword)); + return expandToNode` export const ${config.projectName}Terminals = { ${joinToNode(Object.entries(collection), ([name, regexp]) => `${name}: ${regexp.toString()},`, { appendNewLineIfNotEmpty: true })} }; export type ${config.projectName}TerminalNames = keyof typeof ${config.projectName}Terminals; + + export type ${config.projectName}KeywordNames = ${keywordStrings.length > 0 ? keywordStrings.join(' | ') : 'never'}; + + export type ${config.projectName}TokenNames = ${config.projectName}TerminalNames | ${config.projectName}KeywordNames; `.appendNewLine(); } diff --git a/packages/langium-cli/test/generator/ast-generator.test.ts b/packages/langium-cli/test/generator/ast-generator.test.ts index b15d63346..805b32c51 100644 --- a/packages/langium-cli/test/generator/ast-generator.test.ts +++ b/packages/langium-cli/test/generator/ast-generator.test.ts @@ -461,7 +461,7 @@ function testGeneratedInterface(name: string, grammar: string, expected: string) } function testGeneratedAst(name: string, grammar: string, expected: string): void { - testGenerated(name, grammar, expected, 'export type', 'export type testAstType', 1); + testGenerated(name, grammar, expected, 'export type', 'export type testAstType', 3); } function testTypeMetaData(name: string, grammar: string, expected: string): void { From 79dc94c57955a69be0ac95f3c3f4f26a5cb0de12 Mon Sep 17 00:00:00 2001 From: Abdelrahman Abounegm Date: Sun, 25 Aug 2024 22:35:22 +0300 Subject: [PATCH 2/4] Update generated ASTs --- examples/arithmetics/src/language-server/generated/ast.ts | 4 ++++ examples/domainmodel/src/language-server/generated/ast.ts | 4 ++++ examples/requirements/src/language-server/generated/ast.ts | 4 ++++ examples/statemachine/src/language-server/generated/ast.ts | 4 ++++ packages/langium/src/languages/generated/ast.ts | 4 ++++ 5 files changed, 20 insertions(+) diff --git a/examples/arithmetics/src/language-server/generated/ast.ts b/examples/arithmetics/src/language-server/generated/ast.ts index 915440201..a3ce178ad 100644 --- a/examples/arithmetics/src/language-server/generated/ast.ts +++ b/examples/arithmetics/src/language-server/generated/ast.ts @@ -17,6 +17,10 @@ export const ArithmeticsTerminals = { export type ArithmeticsTerminalNames = keyof typeof ArithmeticsTerminals; +export type ArithmeticsKeywordNames = "%" | "(" | ")" | "*" | "+" | "," | "-" | "/" | ":" | ";" | "^" | "def" | "module"; + +export type ArithmeticsTokenNames = ArithmeticsTerminalNames | ArithmeticsKeywordNames; + export type AbstractDefinition = DeclaredParameter | Definition; export const AbstractDefinition = 'AbstractDefinition'; diff --git a/examples/domainmodel/src/language-server/generated/ast.ts b/examples/domainmodel/src/language-server/generated/ast.ts index b207fec0a..664058bb1 100644 --- a/examples/domainmodel/src/language-server/generated/ast.ts +++ b/examples/domainmodel/src/language-server/generated/ast.ts @@ -16,6 +16,10 @@ export const DomainModelTerminals = { export type DomainModelTerminalNames = keyof typeof DomainModelTerminals; +export type DomainModelKeywordNames = "." | ":" | "datatype" | "entity" | "extends" | "many" | "package" | "{" | "}"; + +export type DomainModelTokenNames = DomainModelTerminalNames | DomainModelKeywordNames; + export type AbstractElement = PackageDeclaration | Type; export const AbstractElement = 'AbstractElement'; diff --git a/examples/requirements/src/language-server/generated/ast.ts b/examples/requirements/src/language-server/generated/ast.ts index 85d8671e6..6ab1c8303 100644 --- a/examples/requirements/src/language-server/generated/ast.ts +++ b/examples/requirements/src/language-server/generated/ast.ts @@ -17,6 +17,10 @@ export const RequirementsAndTestsTerminals = { export type RequirementsAndTestsTerminalNames = keyof typeof RequirementsAndTestsTerminals; +export type RequirementsAndTestsKeywordNames = "," | ":" | "applicable" | "contact" | "environment" | "for" | "req" | "," | ":" | "=" | "applicable" | "contact" | "for" | "testFile" | "tests" | "tst"; + +export type RequirementsAndTestsTokenNames = RequirementsAndTestsTerminalNames | RequirementsAndTestsKeywordNames; + export interface Contact extends AstNode { readonly $container: RequirementModel | TestModel; readonly $type: 'Contact'; diff --git a/examples/statemachine/src/language-server/generated/ast.ts b/examples/statemachine/src/language-server/generated/ast.ts index a191ba01d..a98f93aff 100644 --- a/examples/statemachine/src/language-server/generated/ast.ts +++ b/examples/statemachine/src/language-server/generated/ast.ts @@ -16,6 +16,10 @@ export const StatemachineTerminals = { export type StatemachineTerminalNames = keyof typeof StatemachineTerminals; +export type StatemachineKeywordNames = "=>" | "actions" | "commands" | "end" | "events" | "initialState" | "state" | "statemachine" | "{" | "}"; + +export type StatemachineTokenNames = StatemachineTerminalNames | StatemachineKeywordNames; + export interface Command extends AstNode { readonly $container: Statemachine; readonly $type: 'Command'; diff --git a/packages/langium/src/languages/generated/ast.ts b/packages/langium/src/languages/generated/ast.ts index 37ffbbf71..f1246663b 100644 --- a/packages/langium/src/languages/generated/ast.ts +++ b/packages/langium/src/languages/generated/ast.ts @@ -19,6 +19,10 @@ export const LangiumGrammarTerminals = { export type LangiumGrammarTerminalNames = keyof typeof LangiumGrammarTerminals; +export type LangiumGrammarKeywordNames = "!" | "&" | "(" | ")" | "*" | "+" | "+=" | "," | "->" | "." | ".." | ":" | ";" | "<" | "=" | "=>" | ">" | "?" | "?!" | "? Date: Sun, 25 Aug 2024 22:45:43 +0300 Subject: [PATCH 3/4] Improve type-safety of indentation-aware options --- .../langium/src/parser/indentation-aware.ts | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/packages/langium/src/parser/indentation-aware.ts b/packages/langium/src/parser/indentation-aware.ts index 3fe6ccb8b..a62f17821 100644 --- a/packages/langium/src/parser/indentation-aware.ts +++ b/packages/langium/src/parser/indentation-aware.ts @@ -15,7 +15,7 @@ import { DefaultLexer, isTokenTypeArray } from './lexer.js'; type IndentationAwareDelimiter = [begin: TokenName, end: TokenName]; -export interface IndentationTokenBuilderOptions { +export interface IndentationTokenBuilderOptions { /** * The name of the token used to denote indentation in the grammar. * A possible definition in the grammar could look like this: @@ -25,7 +25,7 @@ export interface IndentationTokenBuilderOptions> + ignoreIndentationDelimeters: Array> } export const indentationBuilderDefaultOptions: IndentationTokenBuilderOptions = { @@ -73,15 +73,19 @@ export enum LexingMode { * A token builder that is sensitive to indentation in the input text. * It will generate tokens for indentation and dedentation based on the indentation level. * + * The first generic parameter corresponds to the names of terminal tokens, + * while the second one corresonds to the names of keyword tokens. + * Both parameters are optional and can be imported from `./generated/ast.js`. + * * Inspired by https://github.com/chevrotain/chevrotain/blob/master/examples/lexer/python_indentation/python_indentation.js */ -export class IndentationAwareTokenBuilder extends DefaultTokenBuilder { +export class IndentationAwareTokenBuilder extends DefaultTokenBuilder { /** * The stack in which all the previous matched indentation levels are stored * to understand how deep a the next tokens are nested. */ protected indentationStack: number[] = [0]; - readonly options: IndentationTokenBuilderOptions; + readonly options: IndentationTokenBuilderOptions; /** * The token type to be used for indentation tokens @@ -99,10 +103,10 @@ export class IndentationAwareTokenBuilder ext */ protected whitespaceRegExp = /[ \t]+/y; - constructor(options: Partial>> = indentationBuilderDefaultOptions as IndentationTokenBuilderOptions) { + constructor(options: Partial, NoInfer>> = indentationBuilderDefaultOptions as IndentationTokenBuilderOptions) { super(); this.options = { - ...indentationBuilderDefaultOptions as IndentationTokenBuilderOptions, + ...indentationBuilderDefaultOptions as IndentationTokenBuilderOptions, ...options, }; From 5907cce58e60e0260167bb9c8422017f7ea52cf9 Mon Sep 17 00:00:00 2001 From: Mark Sujew Date: Thu, 29 Aug 2024 11:57:01 +0000 Subject: [PATCH 4/4] Apply set and sorting --- .../src/language-server/generated/ast.ts | 15 +++++- .../src/language-server/generated/ast.ts | 11 +++- .../src/language-server/generated/ast.ts | 13 ++++- .../src/language-server/generated/ast.ts | 12 ++++- .../src/generator/ast-generator.ts | 13 ++--- .../langium/src/languages/generated/ast.ts | 52 ++++++++++++++++++- 6 files changed, 105 insertions(+), 11 deletions(-) diff --git a/examples/arithmetics/src/language-server/generated/ast.ts b/examples/arithmetics/src/language-server/generated/ast.ts index a3ce178ad..7b96eae0d 100644 --- a/examples/arithmetics/src/language-server/generated/ast.ts +++ b/examples/arithmetics/src/language-server/generated/ast.ts @@ -17,7 +17,20 @@ export const ArithmeticsTerminals = { export type ArithmeticsTerminalNames = keyof typeof ArithmeticsTerminals; -export type ArithmeticsKeywordNames = "%" | "(" | ")" | "*" | "+" | "," | "-" | "/" | ":" | ";" | "^" | "def" | "module"; +export type ArithmeticsKeywordNames = + | "%" + | "(" + | ")" + | "*" + | "+" + | "," + | "-" + | "/" + | ":" + | ";" + | "^" + | "def" + | "module"; export type ArithmeticsTokenNames = ArithmeticsTerminalNames | ArithmeticsKeywordNames; diff --git a/examples/domainmodel/src/language-server/generated/ast.ts b/examples/domainmodel/src/language-server/generated/ast.ts index 664058bb1..f777fde95 100644 --- a/examples/domainmodel/src/language-server/generated/ast.ts +++ b/examples/domainmodel/src/language-server/generated/ast.ts @@ -16,7 +16,16 @@ export const DomainModelTerminals = { export type DomainModelTerminalNames = keyof typeof DomainModelTerminals; -export type DomainModelKeywordNames = "." | ":" | "datatype" | "entity" | "extends" | "many" | "package" | "{" | "}"; +export type DomainModelKeywordNames = + | "." + | ":" + | "datatype" + | "entity" + | "extends" + | "many" + | "package" + | "{" + | "}"; export type DomainModelTokenNames = DomainModelTerminalNames | DomainModelKeywordNames; diff --git a/examples/requirements/src/language-server/generated/ast.ts b/examples/requirements/src/language-server/generated/ast.ts index 6ab1c8303..bca51919c 100644 --- a/examples/requirements/src/language-server/generated/ast.ts +++ b/examples/requirements/src/language-server/generated/ast.ts @@ -17,7 +17,18 @@ export const RequirementsAndTestsTerminals = { export type RequirementsAndTestsTerminalNames = keyof typeof RequirementsAndTestsTerminals; -export type RequirementsAndTestsKeywordNames = "," | ":" | "applicable" | "contact" | "environment" | "for" | "req" | "," | ":" | "=" | "applicable" | "contact" | "for" | "testFile" | "tests" | "tst"; +export type RequirementsAndTestsKeywordNames = + | "," + | ":" + | "=" + | "applicable" + | "contact" + | "environment" + | "for" + | "req" + | "testFile" + | "tests" + | "tst"; export type RequirementsAndTestsTokenNames = RequirementsAndTestsTerminalNames | RequirementsAndTestsKeywordNames; diff --git a/examples/statemachine/src/language-server/generated/ast.ts b/examples/statemachine/src/language-server/generated/ast.ts index a98f93aff..7c0df73e3 100644 --- a/examples/statemachine/src/language-server/generated/ast.ts +++ b/examples/statemachine/src/language-server/generated/ast.ts @@ -16,7 +16,17 @@ export const StatemachineTerminals = { export type StatemachineTerminalNames = keyof typeof StatemachineTerminals; -export type StatemachineKeywordNames = "=>" | "actions" | "commands" | "end" | "events" | "initialState" | "state" | "statemachine" | "{" | "}"; +export type StatemachineKeywordNames = + | "=>" + | "actions" + | "commands" + | "end" + | "events" + | "initialState" + | "state" + | "statemachine" + | "{" + | "}"; export type StatemachineTokenNames = StatemachineTerminalNames | StatemachineKeywordNames; diff --git a/packages/langium-cli/src/generator/ast-generator.ts b/packages/langium-cli/src/generator/ast-generator.ts index ba93abe93..4b55c325b 100644 --- a/packages/langium-cli/src/generator/ast-generator.ts +++ b/packages/langium-cli/src/generator/ast-generator.ts @@ -4,7 +4,7 @@ * terms of the MIT License, which is available in the project root. ******************************************************************************/ import type { Grammar, LangiumCoreServices } from 'langium'; -import { type Generated, expandToNode, joinToNode, toString } from 'langium/generate'; +import { EOL, type Generated, expandToNode, joinToNode, toString } from 'langium/generate'; import type { AstTypes, Property, PropertyDefaultValue } from 'langium/grammar'; import type { LangiumConfig } from '../package-types.js'; import { AstUtils, MultiMap, GrammarAST } from 'langium'; @@ -231,15 +231,16 @@ function groupBySupertypes(astTypes: AstTypes): MultiMap { function generateTerminalConstants(grammars: Grammar[], config: LangiumConfig): Generated { let collection: Record = {}; - const keywordTokens: string[] = []; + const keywordTokens = new Set(); grammars.forEach(grammar => { const terminalConstants = collectTerminalRegexps(grammar); collection = {...collection, ...terminalConstants}; - const keywords = collectKeywords(grammar); - keywordTokens.push(...keywords); + for (const keyword of collectKeywords(grammar)) { + keywordTokens.add(keyword); + } }); - const keywordStrings = keywordTokens.map((keyword) => JSON.stringify(keyword)); + const keywordStrings = Array.from(keywordTokens).sort().map((keyword) => JSON.stringify(keyword)); return expandToNode` export const ${config.projectName}Terminals = { @@ -248,7 +249,7 @@ function generateTerminalConstants(grammars: Grammar[], config: LangiumConfig): export type ${config.projectName}TerminalNames = keyof typeof ${config.projectName}Terminals; - export type ${config.projectName}KeywordNames = ${keywordStrings.length > 0 ? keywordStrings.join(' | ') : 'never'}; + export type ${config.projectName}KeywordNames = ${keywordStrings.length > 0 ? keywordStrings.map(keyword => `${EOL} | ${keyword}`).join('') : 'never'}; export type ${config.projectName}TokenNames = ${config.projectName}TerminalNames | ${config.projectName}KeywordNames; `.appendNewLine(); diff --git a/packages/langium/src/languages/generated/ast.ts b/packages/langium/src/languages/generated/ast.ts index f1246663b..41219f074 100644 --- a/packages/langium/src/languages/generated/ast.ts +++ b/packages/langium/src/languages/generated/ast.ts @@ -19,7 +19,57 @@ export const LangiumGrammarTerminals = { export type LangiumGrammarTerminalNames = keyof typeof LangiumGrammarTerminals; -export type LangiumGrammarKeywordNames = "!" | "&" | "(" | ")" | "*" | "+" | "+=" | "," | "->" | "." | ".." | ":" | ";" | "<" | "=" | "=>" | ">" | "?" | "?!" | "?" + | "." + | ".." + | ":" + | ";" + | "<" + | "=" + | "=>" + | ">" + | "?" + | "?!" + | "?