From 5b02bfc397749c81bf7cf37b4b0493a7d93a7781 Mon Sep 17 00:00:00 2001 From: Markus Rudolph Date: Tue, 26 Sep 2023 15:54:06 +0200 Subject: [PATCH] Add EOF token to grammar (#1162) --- packages/langium/src/grammar/generated/ast.ts | 16 +- .../langium/src/grammar/generated/grammar.ts | 211 ++++++++++-------- .../src/grammar/langium-grammar.langium | 7 +- .../langium/src/parser/parser-builder-base.ts | 7 +- packages/langium/src/parser/token-builder.ts | 17 +- .../parser/langium-parser-builder.test.ts | 59 ++++- 6 files changed, 216 insertions(+), 101 deletions(-) diff --git a/packages/langium/src/grammar/generated/ast.ts b/packages/langium/src/grammar/generated/ast.ts index d2ec83a7c..1b70b9998 100644 --- a/packages/langium/src/grammar/generated/ast.ts +++ b/packages/langium/src/grammar/generated/ast.ts @@ -61,7 +61,7 @@ export function isTypeDefinition(item: unknown): item is TypeDefinition { } export interface AbstractElement extends AstNode { - readonly $type: 'AbstractElement' | 'Action' | 'Alternatives' | 'Assignment' | 'CharacterRange' | 'CrossReference' | 'Group' | 'Keyword' | 'NegatedToken' | 'RegexToken' | 'RuleCall' | 'TerminalAlternatives' | 'TerminalGroup' | 'TerminalRuleCall' | 'UnorderedGroup' | 'UntilToken' | 'Wildcard'; + readonly $type: 'AbstractElement' | 'Action' | 'Alternatives' | 'Assignment' | 'CharacterRange' | 'CrossReference' | 'EndOfFile' | 'Group' | 'Keyword' | 'NegatedToken' | 'RegexToken' | 'RuleCall' | 'TerminalAlternatives' | 'TerminalGroup' | 'TerminalRuleCall' | 'UnorderedGroup' | 'UntilToken' | 'Wildcard'; cardinality?: '*' | '+' | '?' lookahead?: '?!' | '?=' } @@ -407,6 +407,16 @@ export function isCrossReference(item: unknown): item is CrossReference { return reflection.isInstance(item, CrossReference); } +export interface EndOfFile extends AbstractElement { + readonly $type: 'EndOfFile'; +} + +export const EndOfFile = 'EndOfFile'; + +export function isEndOfFile(item: unknown): item is EndOfFile { + return reflection.isInstance(item, EndOfFile); +} + export interface Group extends AbstractElement { readonly $type: 'Group'; elements: Array @@ -543,6 +553,7 @@ export type LangiumGrammarAstType = { Conjunction: Conjunction CrossReference: CrossReference Disjunction: Disjunction + EndOfFile: EndOfFile Grammar: Grammar GrammarImport: GrammarImport Group: Group @@ -577,7 +588,7 @@ export type LangiumGrammarAstType = { export class LangiumGrammarAstReflection extends AbstractAstReflection { getAllTypes(): string[] { - return ['AbstractElement', 'AbstractRule', 'AbstractType', 'Action', 'Alternatives', 'ArrayType', 'Assignment', 'CharacterRange', 'Condition', 'Conjunction', 'CrossReference', 'Disjunction', 'Grammar', 'GrammarImport', 'Group', 'InferredType', 'Interface', 'Keyword', 'LiteralCondition', 'NamedArgument', 'NegatedToken', 'Negation', 'Parameter', 'ParameterReference', 'ParserRule', 'ReferenceType', 'RegexToken', 'ReturnType', 'RuleCall', 'SimpleType', 'TerminalAlternatives', 'TerminalGroup', 'TerminalRule', 'TerminalRuleCall', 'Type', 'TypeAttribute', 'TypeDefinition', 'UnionType', 'UnorderedGroup', 'UntilToken', 'Wildcard']; + return ['AbstractElement', 'AbstractRule', 'AbstractType', 'Action', 'Alternatives', 'ArrayType', 'Assignment', 'CharacterRange', 'Condition', 'Conjunction', 'CrossReference', 'Disjunction', 'EndOfFile', 'Grammar', 'GrammarImport', 'Group', 'InferredType', 'Interface', 'Keyword', 'LiteralCondition', 'NamedArgument', 'NegatedToken', 'Negation', 'Parameter', 'ParameterReference', 'ParserRule', 'ReferenceType', 'RegexToken', 'ReturnType', 'RuleCall', 'SimpleType', 'TerminalAlternatives', 'TerminalGroup', 'TerminalRule', 'TerminalRuleCall', 'Type', 'TypeAttribute', 'TypeDefinition', 'UnionType', 'UnorderedGroup', 'UntilToken', 'Wildcard']; } protected override computeIsSubtype(subtype: string, supertype: string): boolean { @@ -589,6 +600,7 @@ export class LangiumGrammarAstReflection extends AbstractAstReflection { case Assignment: case CharacterRange: case CrossReference: + case EndOfFile: case Group: case Keyword: case NegatedToken: diff --git a/packages/langium/src/grammar/generated/grammar.ts b/packages/langium/src/grammar/generated/grammar.ts index 7f8fa1626..8f27cc6f8 100644 --- a/packages/langium/src/grammar/generated/grammar.ts +++ b/packages/langium/src/grammar/generated/grammar.ts @@ -38,7 +38,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] } @@ -62,7 +62,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] }, @@ -88,7 +88,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] }, @@ -132,7 +132,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] }, @@ -158,7 +158,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] }, @@ -261,7 +261,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] } @@ -285,7 +285,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] }, @@ -311,7 +311,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] }, @@ -394,7 +394,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@58" + "$ref": "#/rules@59" }, "arguments": [] } @@ -677,7 +677,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] }, @@ -703,7 +703,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@60" + "$ref": "#/rules@61" }, "arguments": [] } @@ -774,7 +774,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] } @@ -825,7 +825,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar { "$type": "RuleCall", "rule": { - "$ref": "#/rules@46" + "$ref": "#/rules@47" }, "arguments": [] } @@ -855,7 +855,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@60" + "$ref": "#/rules@61" }, "arguments": [] } @@ -945,7 +945,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] }, @@ -1023,7 +1023,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] }, @@ -1049,7 +1049,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] }, @@ -1155,7 +1155,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] } @@ -1182,7 +1182,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] } @@ -1259,7 +1259,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] } @@ -1372,7 +1372,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@29" + "$ref": "#/rules@30" }, "arguments": [] } @@ -1571,7 +1571,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar { "$type": "RuleCall", "rule": { - "$ref": "#/rules@37" + "$ref": "#/rules@38" }, "arguments": [] }, @@ -1652,7 +1652,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] }, @@ -1696,7 +1696,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@58" + "$ref": "#/rules@59" }, "arguments": [] } @@ -1752,42 +1752,49 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar { "$type": "RuleCall", "rule": { - "$ref": "#/rules@25" + "$ref": "#/rules@26" }, "arguments": [] }, { "$type": "RuleCall", "rule": { - "$ref": "#/rules@26" + "$ref": "#/rules@27" }, "arguments": [] }, { "$type": "RuleCall", "rule": { - "$ref": "#/rules@43" + "$ref": "#/rules@44" }, "arguments": [] }, { "$type": "RuleCall", "rule": { - "$ref": "#/rules@35" + "$ref": "#/rules@36" }, "arguments": [] }, { "$type": "RuleCall", "rule": { - "$ref": "#/rules@36" + "$ref": "#/rules@37" }, "arguments": [] }, { "$type": "RuleCall", "rule": { - "$ref": "#/rules@44" + "$ref": "#/rules@45" + }, + "arguments": [] + }, + { + "$type": "RuleCall", + "rule": { + "$ref": "#/rules@25" }, "arguments": [] } @@ -1800,6 +1807,32 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "parameters": [], "wildcard": false }, + { + "$type": "ParserRule", + "name": "EndOfFile", + "definition": { + "$type": "Group", + "elements": [ + { + "$type": "Action", + "inferredType": { + "$type": "InferredType", + "name": "EndOfFile" + } + }, + { + "$type": "Keyword", + "value": "EOF" + } + ] + }, + "definesHiddenTokens": false, + "entry": false, + "fragment": false, + "hiddenTokens": [], + "parameters": [], + "wildcard": false + }, { "$type": "ParserRule", "name": "Keyword", @@ -1810,7 +1843,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@60" + "$ref": "#/rules@61" }, "arguments": [] } @@ -1840,7 +1873,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] }, @@ -1861,7 +1894,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@27" + "$ref": "#/rules@28" }, "arguments": [] } @@ -1880,7 +1913,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@27" + "$ref": "#/rules@28" }, "arguments": [] } @@ -1925,7 +1958,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] }, @@ -1951,7 +1984,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@29" + "$ref": "#/rules@30" }, "arguments": [] } @@ -2006,7 +2039,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar { "$type": "RuleCall", "rule": { - "$ref": "#/rules@30" + "$ref": "#/rules@31" }, "arguments": [] }, @@ -2033,7 +2066,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@30" + "$ref": "#/rules@31" }, "arguments": [] } @@ -2063,7 +2096,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar { "$type": "RuleCall", "rule": { - "$ref": "#/rules@31" + "$ref": "#/rules@32" }, "arguments": [] }, @@ -2090,7 +2123,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@31" + "$ref": "#/rules@32" }, "arguments": [] } @@ -2120,7 +2153,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar { "$type": "RuleCall", "rule": { - "$ref": "#/rules@32" + "$ref": "#/rules@33" }, "arguments": [] }, @@ -2145,7 +2178,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@31" + "$ref": "#/rules@32" }, "arguments": [] } @@ -2174,21 +2207,21 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar { "$type": "RuleCall", "rule": { - "$ref": "#/rules@34" + "$ref": "#/rules@35" }, "arguments": [] }, { "$type": "RuleCall", "rule": { - "$ref": "#/rules@33" + "$ref": "#/rules@34" }, "arguments": [] }, { "$type": "RuleCall", "rule": { - "$ref": "#/rules@28" + "$ref": "#/rules@29" }, "arguments": [] } @@ -2218,7 +2251,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar { "$type": "RuleCall", "rule": { - "$ref": "#/rules@29" + "$ref": "#/rules@30" }, "arguments": [] }, @@ -2250,7 +2283,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] }, @@ -2294,7 +2327,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@60" + "$ref": "#/rules@61" }, "arguments": [] } @@ -2343,7 +2376,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] }, @@ -2364,7 +2397,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@27" + "$ref": "#/rules@28" }, "arguments": [] } @@ -2383,7 +2416,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@27" + "$ref": "#/rules@28" }, "arguments": [] } @@ -2445,7 +2478,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@58" + "$ref": "#/rules@59" }, "arguments": [] } @@ -2479,7 +2512,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@38" + "$ref": "#/rules@39" }, "arguments": [] } @@ -2506,28 +2539,28 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar { "$type": "RuleCall", "rule": { - "$ref": "#/rules@25" + "$ref": "#/rules@26" }, "arguments": [] }, { "$type": "RuleCall", "rule": { - "$ref": "#/rules@26" + "$ref": "#/rules@27" }, "arguments": [] }, { "$type": "RuleCall", "rule": { - "$ref": "#/rules@39" + "$ref": "#/rules@40" }, "arguments": [] }, { "$type": "RuleCall", "rule": { - "$ref": "#/rules@41" + "$ref": "#/rules@42" }, "arguments": [] } @@ -2557,7 +2590,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar { "$type": "RuleCall", "rule": { - "$ref": "#/rules@40" + "$ref": "#/rules@41" }, "arguments": [] }, @@ -2587,7 +2620,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar { "$type": "RuleCall", "rule": { - "$ref": "#/rules@38" + "$ref": "#/rules@39" }, "arguments": [] }, @@ -2617,7 +2650,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@38" + "$ref": "#/rules@39" }, "arguments": [] } @@ -2698,7 +2731,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@42" + "$ref": "#/rules@43" }, "arguments": [] } @@ -2732,14 +2765,14 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar { "$type": "RuleCall", "rule": { - "$ref": "#/rules@25" + "$ref": "#/rules@26" }, "arguments": [] }, { "$type": "RuleCall", "rule": { - "$ref": "#/rules@26" + "$ref": "#/rules@27" }, "arguments": [] } @@ -2858,7 +2891,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] } @@ -2914,7 +2947,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] } @@ -2931,7 +2964,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] } @@ -2950,7 +2983,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@45" + "$ref": "#/rules@46" }, "arguments": [] } @@ -2973,7 +3006,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@47" + "$ref": "#/rules@48" }, "arguments": [] } @@ -3004,7 +3037,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar { "$type": "RuleCall", "rule": { - "$ref": "#/rules@48" + "$ref": "#/rules@49" }, "arguments": [] }, @@ -3031,7 +3064,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@48" + "$ref": "#/rules@49" }, "arguments": [] } @@ -3061,7 +3094,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar { "$type": "RuleCall", "rule": { - "$ref": "#/rules@49" + "$ref": "#/rules@50" }, "arguments": [] }, @@ -3084,7 +3117,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@49" + "$ref": "#/rules@50" }, "arguments": [] }, @@ -3115,7 +3148,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar { "$type": "RuleCall", "rule": { - "$ref": "#/rules@50" + "$ref": "#/rules@51" }, "arguments": [] }, @@ -3164,49 +3197,49 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar { "$type": "RuleCall", "rule": { - "$ref": "#/rules@57" + "$ref": "#/rules@58" }, "arguments": [] }, { "$type": "RuleCall", "rule": { - "$ref": "#/rules@52" + "$ref": "#/rules@53" }, "arguments": [] }, { "$type": "RuleCall", "rule": { - "$ref": "#/rules@51" + "$ref": "#/rules@52" }, "arguments": [] }, { "$type": "RuleCall", "rule": { - "$ref": "#/rules@53" + "$ref": "#/rules@54" }, "arguments": [] }, { "$type": "RuleCall", "rule": { - "$ref": "#/rules@54" + "$ref": "#/rules@55" }, "arguments": [] }, { "$type": "RuleCall", "rule": { - "$ref": "#/rules@55" + "$ref": "#/rules@56" }, "arguments": [] }, { "$type": "RuleCall", "rule": { - "$ref": "#/rules@56" + "$ref": "#/rules@57" }, "arguments": [] } @@ -3255,7 +3288,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar { "$type": "RuleCall", "rule": { - "$ref": "#/rules@47" + "$ref": "#/rules@48" }, "arguments": [] }, @@ -3296,12 +3329,12 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "CrossReference", "type": { - "$ref": "#/rules@46" + "$ref": "#/rules@47" }, "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] }, @@ -3345,7 +3378,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@50" + "$ref": "#/rules@51" }, "arguments": [] } @@ -3387,7 +3420,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@50" + "$ref": "#/rules@51" }, "arguments": [] } @@ -3425,7 +3458,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@61" + "$ref": "#/rules@62" }, "arguments": [] } @@ -3493,7 +3526,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@25" + "$ref": "#/rules@26" }, "arguments": [] } @@ -3512,7 +3545,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar "terminal": { "$type": "RuleCall", "rule": { - "$ref": "#/rules@25" + "$ref": "#/rules@26" }, "arguments": [] } @@ -3610,7 +3643,7 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar { "$type": "RuleCall", "rule": { - "$ref": "#/rules@59" + "$ref": "#/rules@60" }, "arguments": [] } diff --git a/packages/langium/src/grammar/langium-grammar.langium b/packages/langium/src/grammar/langium-grammar.langium index 2d122c2c5..806c882f1 100644 --- a/packages/langium/src/grammar/langium-grammar.langium +++ b/packages/langium/src/grammar/langium-grammar.langium @@ -99,7 +99,12 @@ AbstractTerminal infers AbstractElement: ParenthesizedElement | PredicatedKeyword | PredicatedRuleCall | - PredicatedGroup; + PredicatedGroup | + EndOfFile + ; + +EndOfFile: + {infer EndOfFile} 'EOF'; Keyword: value=STRING; diff --git a/packages/langium/src/parser/parser-builder-base.ts b/packages/langium/src/parser/parser-builder-base.ts index 80922fbb3..f58374a37 100644 --- a/packages/langium/src/parser/parser-builder-base.ts +++ b/packages/langium/src/parser/parser-builder-base.ts @@ -9,8 +9,8 @@ import type { AbstractElement, Action, Alternatives, Condition, CrossReference, import type { BaseParser } from './langium-parser.js'; import type { AstNode } from '../syntax-tree.js'; import type { Cardinality } from '../grammar/internal-grammar-util.js'; -import { EMPTY_ALT } from 'chevrotain'; -import { isAction, isAlternatives, isAssignment, isConjunction, isCrossReference, isDisjunction, isGroup, isKeyword, isLiteralCondition, isNegation, isParameterReference, isParserRule, isRuleCall, isTerminalRule, isUnorderedGroup } from '../grammar/generated/ast.js'; +import { EMPTY_ALT, EOF } from 'chevrotain'; +import { isAction, isAlternatives, isEndOfFile, isAssignment, isConjunction, isCrossReference, isDisjunction, isGroup, isKeyword, isLiteralCondition, isNegation, isParameterReference, isParserRule, isRuleCall, isTerminalRule, isUnorderedGroup } from '../grammar/generated/ast.js'; import { assertUnreachable, ErrorWithLocation } from '../utils/errors.js'; import { stream } from '../utils/stream.js'; import { getTypeName } from '../grammar/internal-grammar-util.js'; @@ -88,6 +88,9 @@ function buildElement(ctx: RuleContext, element: AbstractElement, ignoreGuard = method = buildUnorderedGroup(ctx, element); } else if (isGroup(element)) { method = buildGroup(ctx, element); + } else if(isEndOfFile(element)) { + const idx = ctx.consume++; + method = () => ctx.parser.consume(idx, EOF, element); } else { throw new ErrorWithLocation(element.$cstNode, `Unexpected element type: ${element.$type}`); } diff --git a/packages/langium/src/parser/token-builder.ts b/packages/langium/src/parser/token-builder.ts index 6b257fb43..fe93059ac 100644 --- a/packages/langium/src/parser/token-builder.ts +++ b/packages/langium/src/parser/token-builder.ts @@ -7,10 +7,10 @@ import type { CustomPatternMatcherFunc, TokenPattern, TokenType, TokenVocabulary } from 'chevrotain'; import type { AbstractRule, Grammar, Keyword, TerminalRule } from '../grammar/generated/ast.js'; import type { Stream } from '../utils/stream.js'; -import { Lexer } from 'chevrotain'; -import { isKeyword, isParserRule, isTerminalRule } from '../grammar/generated/ast.js'; +import { Lexer, EOF } from 'chevrotain'; +import { isKeyword, isParserRule, isTerminalRule, isEndOfFile } from '../grammar/generated/ast.js'; import { terminalRegex } from '../grammar/internal-grammar-util.js'; -import { streamAllContents } from '../utils/ast-util.js'; +import { streamAllContents, streamAst } from '../utils/ast-util.js'; import { getAllReachableRules } from '../utils/grammar-util.js'; import { getCaseInsensitivePattern, isWhitespaceRegExp, partialMatches } from '../utils/regex-util.js'; import { stream } from '../utils/stream.js'; @@ -38,6 +38,11 @@ export class DefaultTokenBuilder implements TokenBuilder { tokens.push(terminalToken); } }); + + //reminder: EOF should always be the last token, because it is very unlikely that it will be matched within the input + if (reachableRules.some(r => streamAst(r.definition).some(isEndOfFile))) { + tokens.push(EOF); + } return tokens; } @@ -49,16 +54,16 @@ export class DefaultTokenBuilder implements TokenBuilder { protected buildTerminalToken(terminal: TerminalRule): TokenType { const regex = terminalRegex(terminal); const pattern = regex.flags.includes('u') ? this.regexPatternFunction(regex) : regex; - const token: TokenType = { + const tokenType: TokenType = { name: terminal.name, PATTERN: pattern, LINE_BREAKS: true }; if (terminal.hidden) { // Only skip tokens that are able to accept whitespace - token.GROUP = isWhitespaceRegExp(regex) ? Lexer.SKIPPED : 'hidden'; + tokenType.GROUP = isWhitespaceRegExp(regex) ? Lexer.SKIPPED : 'hidden'; } - return token; + return tokenType; } protected regexPatternFunction(regex: RegExp): CustomPatternMatcherFunc { diff --git a/packages/langium/test/parser/langium-parser-builder.test.ts b/packages/langium/test/parser/langium-parser-builder.test.ts index 70b5dec65..95a3315c9 100644 --- a/packages/langium/test/parser/langium-parser-builder.test.ts +++ b/packages/langium/test/parser/langium-parser-builder.test.ts @@ -6,8 +6,10 @@ import type { TokenType, TokenVocabulary } from 'chevrotain'; import type { AstNode, Grammar, GrammarAST, LangiumParser, TokenBuilderOptions } from 'langium'; +import { createLangiumGrammarServices, EmptyFileSystem, createServicesForGrammar, DefaultTokenBuilder} from 'langium'; import { describe, expect, test, onTestFailed, beforeEach } from 'vitest'; -import { createServicesForGrammar, DefaultTokenBuilder } from 'langium'; +import { parseHelper } from 'langium/test'; +import { EOF } from 'chevrotain'; describe('Predicated grammar rules with alternatives', () => { @@ -675,6 +677,61 @@ describe('ALL(*) parser', () => { }); }); +describe('Handling EOF', () => { + test('Use EOF as last part of the entry rule definition', async () => { + const grammar = ` + grammar Test + entry Main: greet='Hello!' EOF; + `; + const services = await createLangiumGrammarServices(EmptyFileSystem); + const output = await parseHelper(services.grammar)(grammar, {validation: true}); + expect(output.parseResult.lexerErrors.length).toBe(0); + expect(output.parseResult.parserErrors.length).toBe(0); + expect(output.diagnostics?.length ?? 0).toBe(0); + }); + + test('Use EOF as a line break', async () => { + const grammar = ` + grammar Test + entry Lines: lines+=Line*; + Line: text=ID (EOL | EOF); + terminal ID: /[_a-zA-Z][\\w_]*/; + terminal EOL: /\\r?\\n/; + `; + const langiumServices = await createLangiumGrammarServices(EmptyFileSystem); + const output = await parseHelper(langiumServices.grammar)(grammar, {validation: true}); + expect(output.parseResult.lexerErrors.length).toBe(0); + expect(output.parseResult.parserErrors.length).toBe(0); + expect(output.diagnostics?.length ?? 0).toBe(0); + + const grammarServices = await createServicesForGrammar({ grammar }); + const parse = parseHelper(grammarServices); + const document = await parse('First\nMiddle\nLast', {validation: true}); + expect(document.parseResult.lexerErrors.length).toBe(0); + expect(document.parseResult.parserErrors.length).toBe(0); + expect(document.diagnostics?.length ?? 0).toBe(0); + }); + + test('Use EOF in an invalid position', async () => { + const grammar = ` + grammar Test + entry Main: greet='Hello!' EOF name='user!'; + `; + const services = await createServicesForGrammar({ grammar }); + const parse = parseHelper(services); + + const document = await parse('Hello!user!', {validation: true}); + expect(document.parseResult.parserErrors.length).toBe(1); + expect(document.parseResult.parserErrors[0].name).toBe('MismatchedTokenException'); + expect(document.parseResult.parserErrors[0].token.tokenType.name).toBe('user!'); + + const second = await parse('Hello!', {validation: true}); + expect(second.parseResult.parserErrors.length).toBe(1); + expect(second.parseResult.parserErrors[0].name).toBe('MismatchedTokenException'); + expect(second.parseResult.parserErrors[0].token.tokenType).toBe(EOF); + }); +}); + async function parserFromGrammar(grammar: string): Promise { return (await createServicesForGrammar({ grammar })).parser.LangiumParser; }