Refactor document factory to support async parsing (#1306)

examples/arithmetics/src/cli/cli-util.ts

@@ -21,7 +21,7 @@ export async function extractDocument<T extends AstNode>(fileName: string, exten
         process.exit(1);
     }
 
-    const document = services.shared.workspace.LangiumDocuments.getOrCreateDocument(URI.file(path.resolve(fileName)));
+    const document = await services.shared.workspace.LangiumDocuments.getOrCreateDocument(URI.file(path.resolve(fileName)));
     await services.shared.workspace.DocumentBuilder.build([document], { validation: true });
 
     const validationErrors = (document.diagnostics ?? []).filter(e => e.severity === 1);

examples/domainmodel/src/cli/cli-util.ts

@@ -22,7 +22,7 @@ export async function extractDocument<T extends AstNode>(fileName: string, exten
         process.exit(1);
     }
 
-    const document = services.shared.workspace.LangiumDocuments.getOrCreateDocument(URI.file(path.resolve(fileName)));
+    const document = await services.shared.workspace.LangiumDocuments.getOrCreateDocument(URI.file(path.resolve(fileName)));
     await services.shared.workspace.DocumentBuilder.build([document], { validation: true });
 
     const validationErrors = (document.diagnostics ?? []).filter(e => e.severity === 1);

examples/domainmodel/src/language-server/domain-model-rename-refactoring.ts

@@ -69,8 +69,11 @@ export class DomainModelRenameProvider extends DefaultRenameProvider {
         return { changes };
     }
 
-    protected hasQualifiedNameText(uri: URI, range: Range) {
-        const langiumDoc = this.langiumDocuments.getOrCreateDocument(uri);
+    protected hasQualifiedNameText(uri: URI, range: Range): boolean {
+        const langiumDoc = this.langiumDocuments.getDocument(uri);
+        if (!langiumDoc) {
+            return false;
+        }
         const rangeText = langiumDoc.textDocument.getText(range);
         return rangeText.includes('.', 0);
     }

examples/requirements/src/cli/cli-util.ts

@@ -54,7 +54,7 @@ export async function extractDocuments(fileName: string, services: LangiumServic
             process.exit(1);
         }
     });
-    const mainDocument = services.shared.workspace.LangiumDocuments.getOrCreateDocument(URI.file(path.resolve(fileName)));
+    const mainDocument = await services.shared.workspace.LangiumDocuments.getOrCreateDocument(URI.file(path.resolve(fileName)));
 
     return [mainDocument, documents];
 }

examples/statemachine/src/cli/cli-util.ts

@@ -21,7 +21,7 @@ export async function extractDocument(fileName: string, extensions: readonly str
         process.exit(1);
     }
 
-    const document = services.shared.workspace.LangiumDocuments.getOrCreateDocument(URI.file(path.resolve(fileName)));
+    const document = await services.shared.workspace.LangiumDocuments.getOrCreateDocument(URI.file(path.resolve(fileName)));
     await services.shared.workspace.DocumentBuilder.build([document], { validation: true });
 
     const validationErrors = (document.diagnostics ?? []).filter(e => e.severity === 1);

packages/generator-langium/templates/cli/src/cli/cli-util.ts

@@ -16,7 +16,7 @@ export async function extractDocument(fileName: string, services: LangiumService
         process.exit(1);
     }
 
-    const document = services.shared.workspace.LangiumDocuments.getOrCreateDocument(URI.file(path.resolve(fileName)));
+    const document = await services.shared.workspace.LangiumDocuments.getOrCreateDocument(URI.file(path.resolve(fileName)));
     await services.shared.workspace.DocumentBuilder.build([document], { validation: true });
 
     const validationErrors = (document.diagnostics ?? []).filter(e => e.severity === 1);

packages/langium-cli/src/generate.ts

@@ -9,7 +9,7 @@ import type { LangiumConfig, LangiumLanguageConfig} from './package.js';
 import { URI } from 'langium';
 import { loadConfig } from './package.js';
 import { copyAstNode, getDocument, GrammarAST, linkContentToContainer } from 'langium';
-import { createLangiumGrammarServices, resolveImport, resolveTransitiveImports } from 'langium/grammar';
+import { createLangiumGrammarServices, resolveImport, resolveImportUri, resolveTransitiveImports } from 'langium/grammar';
 import { NodeFileSystem } from 'langium/node';
 import { generateAst } from './generator/ast-generator.js';
 import { serializeGrammar } from './generator/grammar-serializer.js';
@@ -123,17 +123,17 @@ type GrammarElement = GrammarAST.AbstractRule | GrammarAST.Type | GrammarAST.Int
 const { shared: sharedServices, grammar: grammarServices } = createLangiumGrammarServices(NodeFileSystem);
 const documents = sharedServices.workspace.LangiumDocuments;
 
-function eagerLoad(document: LangiumDocument, uris: Set<string> = new Set()): URI[] {
+async function eagerLoad(document: LangiumDocument, uris: Set<string> = new Set()): Promise<URI[]> {
     const uriString = document.uri.toString();
     if (!uris.has(uriString)) {
         uris.add(uriString);
         const grammar = document.parseResult.value;
         if (GrammarAST.isGrammar(grammar)) {
             for (const imp of grammar.imports) {
-                const importedGrammar = resolveImport(documents, imp);
-                if (importedGrammar) {
-                    const importedDoc = getDocument(importedGrammar);
-                    eagerLoad(importedDoc, uris);
+                const importUri = resolveImportUri(imp);
+                if (importUri) {
+                    const document = await sharedServices.workspace.LangiumDocuments.getOrCreateDocument(importUri);
+                    await eagerLoad(document, uris);
                 }
             }
         }
@@ -229,8 +229,8 @@ async function buildAll(config: LangiumConfig): Promise<Map<string, LangiumDocum
     const uris = new Set<string>();
     for (const languageConfig of config.languages) {
         const absGrammarPath = URI.file(path.resolve(relPath, languageConfig.grammar));
-        const document = documents.getOrCreateDocument(absGrammarPath);
-        eagerLoad(document, uris);
+        const document = await documents.getOrCreateDocument(absGrammarPath);
+        await eagerLoad(document, uris);
     }
     for (const doc of documents.all) {
         map.set(doc.uri.fsPath, doc);
@@ -419,8 +419,7 @@ export async function generateTypes(options: ExtractTypesOptions): Promise<void>
             return;
         }
     }
-
-    const grammarDoc = await doLoadAndUpdate(documents.getOrCreateDocument(URI.file(grammarPath)));
+    const grammarDoc = await doLoadAndUpdate(await documents.getOrCreateDocument(URI.file(grammarPath)));
     const genTypes = generateTypesFile(grammarServices, [grammarDoc.parseResult.value as Grammar]);
     await writeWithFail(typesFilePath, genTypes, { watch: false });
     log('log', { watch: false }, `Generated type definitions to: ${chalk.white.bold(typesFilePath)}`);
@@ -431,7 +430,7 @@ export async function generateTypes(options: ExtractTypesOptions): Promise<void>
  * Builds the given grammar document and all imported grammars.
  */
 async function doLoadAndUpdate(grammarDoc: LangiumDocument): Promise<LangiumDocument> {
-    const allUris = eagerLoad(grammarDoc);
+    const allUris = await eagerLoad(grammarDoc);
     await sharedServices.workspace.DocumentBuilder.update(allUris, []);
     for (const doc of documents.all) {
         await sharedServices.workspace.DocumentBuilder.build([doc]);

packages/langium-sprotty/src/diagram-generator.ts

@@ -49,13 +49,13 @@ export abstract class LangiumDiagramGenerator implements IDiagramGenerator {
     /**
      * Builds a `GeneratorContext` and calls `generateRoot` with it.
      */
-    generate(args: LangiumDiagramGeneratorArguments): SModelRoot | Promise<SModelRoot> {
+    async generate(args: LangiumDiagramGeneratorArguments): Promise<SModelRoot> {
         if (!args.document) {
             const sourceUri = args.options.sourceUri as string;
             if (!sourceUri) {
                 return Promise.reject("Missing 'sourceUri' option in request.");
             }
-            args.document = this.langiumDocuments.getOrCreateDocument(URI.parse(sourceUri));
+            args.document = await this.langiumDocuments.getOrCreateDocument(URI.parse(sourceUri));
         }
         if (!args.cancelToken) {
             args.cancelToken = CancellationToken.None;

packages/langium-sprotty/src/trace-provider.ts

@@ -72,7 +72,10 @@ export class DefaultTraceProvider implements TraceProvider {
         }
         try {
             const traceUri = URI.parse(target.trace);
-            const document = this.langiumDocuments.getOrCreateDocument(traceUri.with({ fragment: null, query: null }));
+            const document = this.langiumDocuments.getDocument(traceUri.with({ fragment: null, query: null }));
+            if (!document) {
+                return undefined;
+            }
             return this.astNodeLocator.getAstNode(document.parseResult.value, traceUri.fragment);
         } catch (err) {
             console.warn(`Could not retrieve source of trace: ${target.trace}`, err);

packages/langium-vscode/src/language-server/railroad-handler.ts

@@ -22,10 +22,10 @@ export function registerRailroadConnectionHandler(connection: Connection, servic
     });
     // After receiving the `DOCUMENTS_VALIDATED_NOTIFICATION`
     // the vscode extension will perform the following request
-    connection.onRequest(RAILROAD_DIAGRAM_REQUEST, (uri: string) => {
+    connection.onRequest(RAILROAD_DIAGRAM_REQUEST, async (uri: string) => {
         try {
             const parsedUri = URI.parse(uri);
-            const document = documents.getOrCreateDocument(parsedUri);
+            const document = await documents.getOrCreateDocument(parsedUri);
             if (document.diagnostics?.some(e => e.severity === DiagnosticSeverity.Error)) {
                 return undefined;
             }

packages/langium/src/default-module.ts

@@ -48,6 +48,7 @@ import { DefaultLexer } from './parser/lexer.js';
 import { JSDocDocumentationProvider } from './documentation/documentation-provider.js';
 import { DefaultCommentProvider } from './documentation/comment-provider.js';
 import { LangiumParserErrorMessageProvider } from './parser/langium-parser.js';
+import { DefaultAsyncParser } from './parser/async-parser.js';
 import { DefaultWorkspaceLock } from './workspace/workspace-lock.js';
 
 /**
@@ -68,6 +69,7 @@ export function createDefaultModule(context: DefaultModuleContext): Module<Langi
             DocumentationProvider: (services) => new JSDocDocumentationProvider(services)
         },
         parser: {
+            AsyncParser: (services) => new DefaultAsyncParser(services),
             GrammarConfig: (services) => createGrammarConfig(services),
             LangiumParser: (services) => createLangiumParser(services),
             CompletionParser: (services) => createCompletionParser(services),

packages/langium/src/grammar/internal-grammar-util.ts

@@ -89,16 +89,16 @@ export function resolveImportUri(imp: ast.GrammarImport): URI | undefined {
 
 export function resolveImport(documents: LangiumDocuments, imp: ast.GrammarImport): ast.Grammar | undefined {
     const resolvedUri = resolveImportUri(imp);
-    try {
-        if (resolvedUri) {
-            const resolvedDocument = documents.getOrCreateDocument(resolvedUri);
-            const node = resolvedDocument.parseResult.value;
-            if (ast.isGrammar(node)) {
-                return node;
-            }
-        }
-    } catch {
-        // NOOP
+    if (!resolvedUri) {
+        return undefined;
+    }
+    const resolvedDocument = documents.getDocument(resolvedUri);
+    if (!resolvedDocument) {
+        return undefined;
+    }
+    const node = resolvedDocument.parseResult.value;
+    if (ast.isGrammar(node)) {
+        return node;
     }
     return undefined;
 }

packages/langium/src/grammar/lsp/grammar-call-hierarchy.ts

@@ -15,14 +15,18 @@ import { findLeafNodeAtOffset } from '../../utils/cst-util.js';
 import { isParserRule, isRuleCall } from '../../languages/generated/ast.js';
 
 export class LangiumGrammarCallHierarchyProvider extends AbstractCallHierarchyProvider {
+
     protected getIncomingCalls(node: AstNode, references: Stream<ReferenceDescription>): CallHierarchyIncomingCall[] | undefined {
         if (!isParserRule(node)) {
             return undefined;
         }
         // This map is used to group incoming calls to avoid duplicates.
         const uniqueRules = new Map<string, { parserRule: CstNode, nameNode: CstNode, targetNodes: CstNode[], docUri: string }>();
         references.forEach(ref => {
-            const doc = this.documents.getOrCreateDocument(ref.sourceUri);
+            const doc = this.documents.getDocument(ref.sourceUri);
+            if (!doc) {
+                return;
+            }
             const rootNode = doc.parseResult.value;
             if (!rootNode.$cstNode) {
                 return;

packages/langium/src/grammar/lsp/grammar-type-hierarchy.ts

@@ -37,7 +37,11 @@ export class LangiumGrammarTypeHierarchyProvider extends AbstractTypeHierarchyPr
         const items = this.references
             .findReferences(node, {includeDeclaration: false})
             .flatMap(ref => {
-                const document = this.documents.getOrCreateDocument(ref.sourceUri);
+                const document = this.documents.getDocument(ref.sourceUri);
+                if (!document) {
+                    return [];
+                }
+
                 const rootNode = document.parseResult.value;
                 if (!rootNode.$cstNode) {
                     return [];

packages/langium/src/grammar/references/grammar-references.ts

@@ -154,13 +154,15 @@ export class LangiumGrammarReferences extends DefaultReferences {
     protected findRulesWithReturnType(interf: Interface | Type): Array<ParserRule | Action> {
         const rules: Array<ParserRule | Action> = [];
         const refs = this.index.findAllReferences(interf, this.nodeLocator.getAstNodePath(interf));
-        refs.forEach(ref => {
-            const doc = this.documents.getOrCreateDocument(ref.sourceUri);
-            const astNode = this.nodeLocator.getAstNode(doc.parseResult.value, ref.sourcePath);
-            if (isParserRule(astNode) || isAction(astNode)) {
-                rules.push(astNode);
+        for (const ref of refs) {
+            const doc = this.documents.getDocument(ref.sourceUri);
+            if (doc) {
+                const astNode = this.nodeLocator.getAstNode(doc.parseResult.value, ref.sourcePath);
+                if (isParserRule(astNode) || isAction(astNode)) {
+                    rules.push(astNode);
+                }
             }
-        });
+        }
         return rules;
     }
 }

packages/langium/src/grammar/references/grammar-scope.ts

@@ -77,8 +77,8 @@ export class LangiumGrammarScopeProvider extends DefaultScopeProvider {
             const uri = resolveImportUri(imp0rt);
             if (uri && !importedUris.has(uri.toString())) {
                 importedUris.add(uri.toString());
-                if (this.langiumDocuments.hasDocument(uri)) {
-                    const importedDocument = this.langiumDocuments.getOrCreateDocument(uri);
+                const importedDocument = this.langiumDocuments.getDocument(uri);
+                if (importedDocument) {
                     const rootNode = importedDocument.parseResult.value;
                     if (isGrammar(rootNode)) {
                         this.gatherImports(rootNode, importedUris);

packages/langium/src/grammar/type-system/types-util.ts

@@ -42,8 +42,11 @@ export function collectChildrenTypes(interfaceNode: Interface, references: Refer
     const childrenTypes = new Set<Interface | Type>();
     childrenTypes.add(interfaceNode);
     const refs = references.findReferences(interfaceNode, {});
-    refs.forEach(ref => {
-        const doc = langiumDocuments.getOrCreateDocument(ref.sourceUri);
+    for (const ref of refs) {
+        const doc = langiumDocuments.getDocument(ref.sourceUri);
+        if (!doc) {
+            continue;
+        }
         const astNode = nodeLocator.getAstNode(doc.parseResult.value, ref.sourcePath);
         if (isInterface(astNode)) {
             childrenTypes.add(astNode);
@@ -52,7 +55,7 @@ export function collectChildrenTypes(interfaceNode: Interface, references: Refer
         } else if (astNode && isType(astNode.$container)) {
             childrenTypes.add(astNode.$container);
         }
-    });
+    }
     return childrenTypes;
 }
 

packages/langium/src/lsp/call-hierarchy-provider.ts

@@ -13,6 +13,7 @@ import type { AstNode } from '../syntax-tree.js';
 import type { Stream } from '../utils/stream.js';
 import type { ReferenceDescription } from '../workspace/ast-descriptions.js';
 import type { LangiumDocument, LangiumDocuments } from '../workspace/documents.js';
+import type { MaybePromise } from '../utils/promise-util.js';
 import { SymbolKind } from 'vscode-languageserver';
 import { findDeclarationNodeAtOffset } from '../utils/cst-util.js';
 import { URI } from '../utils/uri-util.js';
@@ -21,11 +22,11 @@ import { URI } from '../utils/uri-util.js';
  * Language-specific service for handling call hierarchy requests.
  */
 export interface CallHierarchyProvider {
-    prepareCallHierarchy(document: LangiumDocument, params: CallHierarchyPrepareParams, cancelToken?: CancellationToken): CallHierarchyItem[] | undefined;
+    prepareCallHierarchy(document: LangiumDocument, params: CallHierarchyPrepareParams, cancelToken?: CancellationToken): MaybePromise<CallHierarchyItem[] | undefined>;
 
-    incomingCalls(params: CallHierarchyIncomingCallsParams, cancelToken?: CancellationToken): CallHierarchyIncomingCall[] | undefined;
+    incomingCalls(params: CallHierarchyIncomingCallsParams, cancelToken?: CancellationToken): MaybePromise<CallHierarchyIncomingCall[] | undefined>;
 
-    outgoingCalls(params: CallHierarchyOutgoingCallsParams, cancelToken?: CancellationToken): CallHierarchyOutgoingCall[] | undefined;
+    outgoingCalls(params: CallHierarchyOutgoingCallsParams, cancelToken?: CancellationToken): MaybePromise<CallHierarchyOutgoingCall[] | undefined>;
 }
 
 export abstract class AbstractCallHierarchyProvider implements CallHierarchyProvider {
@@ -41,7 +42,7 @@ export abstract class AbstractCallHierarchyProvider implements CallHierarchyProv
         this.references = services.references.References;
     }
 
-    prepareCallHierarchy(document: LangiumDocument<AstNode>, params: CallHierarchyPrepareParams): CallHierarchyItem[] | undefined {
+    prepareCallHierarchy(document: LangiumDocument<AstNode>, params: CallHierarchyPrepareParams): MaybePromise<CallHierarchyItem[] | undefined> {
         const rootNode = document.parseResult.value;
         const targetNode = findDeclarationNodeAtOffset(
             rootNode.$cstNode,
@@ -81,8 +82,8 @@ export abstract class AbstractCallHierarchyProvider implements CallHierarchyProv
         return undefined;
     }
 
-    incomingCalls(params: CallHierarchyIncomingCallsParams): CallHierarchyIncomingCall[] | undefined {
-        const document = this.documents.getOrCreateDocument(URI.parse(params.item.uri));
+    async incomingCalls(params: CallHierarchyIncomingCallsParams): Promise<CallHierarchyIncomingCall[] | undefined> {
+        const document = await this.documents.getOrCreateDocument(URI.parse(params.item.uri));
         const rootNode = document.parseResult.value;
         const targetNode = findDeclarationNodeAtOffset(
             rootNode.$cstNode,
@@ -105,10 +106,10 @@ export abstract class AbstractCallHierarchyProvider implements CallHierarchyProv
     /**
      * Override this method to collect the incoming calls for your language
      */
-    protected abstract getIncomingCalls(node: AstNode, references: Stream<ReferenceDescription>): CallHierarchyIncomingCall[] | undefined;
+    protected abstract getIncomingCalls(node: AstNode, references: Stream<ReferenceDescription>): MaybePromise<CallHierarchyIncomingCall[] | undefined>;
 
-    outgoingCalls(params: CallHierarchyOutgoingCallsParams): CallHierarchyOutgoingCall[] | undefined {
-        const document = this.documents.getOrCreateDocument(URI.parse(params.item.uri));
+    async outgoingCalls(params: CallHierarchyOutgoingCallsParams): Promise<CallHierarchyOutgoingCall[] | undefined> {
+        const document = await this.documents.getOrCreateDocument(URI.parse(params.item.uri));
         const rootNode = document.parseResult.value;
         const targetNode = findDeclarationNodeAtOffset(
             rootNode.$cstNode,
@@ -124,5 +125,5 @@ export abstract class AbstractCallHierarchyProvider implements CallHierarchyProv
     /**
      * Override this method to collect the outgoing calls for your language
      */
-    protected abstract getOutgoingCalls(node: AstNode): CallHierarchyOutgoingCall[] | undefined;
+    protected abstract getOutgoingCalls(node: AstNode): MaybePromise<CallHierarchyOutgoingCall[] | undefined>;
 }