JsonSerializer: Support references to other documents

packages/langium-cli/src/generator/grammar-serializer.ts

@@ -4,7 +4,8 @@
  * terms of the MIT License, which is available in the project root.
  ******************************************************************************/
 
-import type { Grammar, LangiumServices } from 'langium';
+import type { URI } from 'vscode-uri';
+import type { Grammar, LangiumServices, Reference } from 'langium';
 import type { LangiumConfig } from '../package.js';
 import { CompositeGeneratorNode, NL, normalizeEOL, toString } from 'langium';
 import { generatedHeader } from './util.js';
@@ -29,9 +30,17 @@ export function serializeGrammar(services: LangiumServices, grammars: Grammar[],
         if (grammar.name) {
             const production = config.mode === 'production';
             const delimiter = production ? "'" : '`';
+            const uriConverter = (uri: URI, ref: Reference) => {
+                // We expect the grammar to be self-contained after the transformations we've done before
+                throw new Error(`Unexpected reference to symbol '${ref.$refText}' in document: ${uri.toString()}`);
+            };
+            const serializedGrammar = services.serializer.JsonSerializer.serialize(grammar, {
+                space: production ? undefined : 2,
+                uriConverter
+            });
             // The json serializer returns strings with \n line delimiter by default
             // We need to translate these line endings to the OS specific line ending
-            const json = normalizeEOL(services.serializer.JsonSerializer.serialize(grammar, production ? undefined : { space: 2 })
+            const json = normalizeEOL(serializedGrammar
                 .replace(/\\/g, '\\\\')
                 .replace(new RegExp(delimiter, 'g'), '\\' + delimiter));
             node.append(

packages/langium-vscode/src/extension.ts

@@ -134,7 +134,7 @@ function decorateText(document: vscode.TextDocument, decorations: vscode.Decorat
     while ((match = regEx.exec(text))) {
         const startIndex = textIndex + match.index;
         const startPos = document.positionAt(startIndex + indentation);
-        const endPos = document.positionAt(startIndex + match[0].trimRight().length);
+        const endPos = document.positionAt(startIndex + match[0].trimEnd().length);
         if (startPos.isBefore(endPos)) {
             const range = new vscode.Range(startPos, endPos);
             decorations.push({ range });
@@ -144,7 +144,7 @@ function decorateText(document: vscode.TextDocument, decorations: vscode.Decorat
 
 // TODO(@@dd): find bug
 function findIndentation(text: string): number {
-    const indents = text.split(/[\r?\n]/g).map(line => line.trimRight()).filter(line => line.length > 0).map(line => line.search(/\S|$/));
+    const indents = text.split(/[\r?\n]/g).map(line => line.trimEnd()).filter(line => line.length > 0).map(line => line.search(/\S|$/));
     const min = indents.length === 0 ? 0 : Math.min(...indents); // min(...[]) = min() = Infinity
     return Math.max(0, min);
 }

packages/langium/src/generator/template-string.ts

@@ -46,7 +46,7 @@ function internalExpandToString(lineSep: string, staticParts: TemplateStringsArr
         .split(NEWLINE_REGEXP)
         .filter(l => l.trim() !== SNLE)
         // whitespace-only lines are empty (preserving leading whitespace)
-        .map(l => l.replace(SNLE, '').trimRight());
+        .map(l => l.replace(SNLE, '').trimEnd());
 
     // in order to nicely handle single line templates with the leading and trailing termintators (``) on separate lines, like
     //   expandToString`foo
@@ -65,7 +65,7 @@ function internalExpandToString(lineSep: string, staticParts: TemplateStringsArr
     lines = containsLeadingLinebreak ? lines.slice(1) : lines;
 
     // .. and drop the last linebreak if it's the last charactor or is followed by white space
-    const containsTrailingLinebreak = lines.length !== 0 && lines[lines.length-1].trimRight().length === 0;
+    const containsTrailingLinebreak = lines.length !== 0 && lines[lines.length-1].trimEnd().length === 0;
     lines = containsTrailingLinebreak ? lines.slice(0, lines.length-1) : lines;
 
     // finds the minimum indentation

packages/langium/src/serializer/json-serializer.ts

@@ -9,19 +9,29 @@ import type { LangiumServices } from '../services.js';
 import type { AstNode, CstNode, GenericAstNode, Reference } from '../syntax-tree.js';
 import type { Mutable } from '../utils/ast-util.js';
 import type { AstNodeLocator } from '../workspace/ast-node-locator.js';
-import type { DocumentSegment } from '../workspace/documents.js';
+import type { DocumentSegment, LangiumDocument, LangiumDocuments } from '../workspace/documents.js';
+import { URI } from 'vscode-uri';
 import { isAstNode, isReference } from '../syntax-tree.js';
 import { getDocument } from '../utils/ast-util.js';
 import { findNodesForProperty } from '../utils/grammar-util.js';
+import { UriUtils } from '../utils/uri-util.js';
 import type { CommentProvider } from '../documentation/comment-provider.js';
 
 export interface JsonSerializeOptions {
+    /** The space parameter for `JSON.stringify`, controlling whether and how to pretty-print the output. */
     space?: string | number;
+    /** Whether to include the `$refText` property for references (the name used to identify the target node). */
     refText?: boolean;
+    /** Whether to include the `$sourceText` property, which holds the full source text from which an AST node was parsed. */
     sourceText?: boolean;
+    /** Whether to include the `$textRegion` property, which holds information to trace AST node properties to their respective source text regions. */
     textRegions?: boolean;
+    /** Whether to include the `$comment` property, which holds comments according to the CommentProvider service. */
     comments?: boolean;
+    /** The replacer parameter for `JSON.stringify`; the default replacer given as parameter should be used to apply basic replacements. */
     replacer?: (key: string, value: unknown, defaultReplacer: (key: string, value: unknown) => unknown) => unknown
+    /** Used to convert and serialize URIs when the target of a cross-reference is in a different document. */
+    uriConverter?: (uri: URI, reference: Reference) => string
 }
 
 /**
@@ -50,7 +60,7 @@ export function isAstNodeWithComment(node: AstNode): node is AstNodeWithComment
  */
 export interface AstNodeRegionWithAssignments extends DocumentSegment {
     /**
-     * A record containing an entry for each assignd property of the AstNode.
+     * A record containing an entry for each assigned property of the AstNode.
      * The key is equal to the property name and the value is an array of the property values'
      * text regions, regardless of whether the property is a single value or list property.
      */
@@ -74,12 +84,18 @@ export interface JsonSerializer {
     /**
      * Deserialize (parse) a JSON `string` into an `AstNode`.
      */
-    deserialize(content: string): AstNode;
+    deserialize<T extends AstNode = AstNode>(content: string): T;
 }
 
+/**
+ * A cross-reference in the serialized JSON representation of an AstNode.
+ */
 interface IntermediateReference {
-    $refText?: string
+    /** URI pointing to the target element. This is either `#${path}` if the target is in the same document, or `${documentURI}#${path}` otherwise. */
     $ref?: string
+    /** The actual text used to look up the reference target in the surrounding scope. */
+    $refText?: string
+    /** If any problem occurred while resolving the reference, it is described by this property. */
     $error?: string
 }
 
@@ -89,12 +105,19 @@ function isIntermediateReference(obj: unknown): obj is IntermediateReference {
 
 export class DefaultJsonSerializer implements JsonSerializer {
 
-    protected ignoreProperties = new Set(['$container', '$containerProperty', '$containerIndex', '$document', '$cstNode']);
+    /** The set of AstNode properties to be ignored by the serializer. */
+    ignoreProperties = new Set(['$container', '$containerProperty', '$containerIndex', '$document', '$cstNode']);
+
+    /** The document that is currently processed by the serializer; this is used by the replacer function.  */
+    protected currentDocument: LangiumDocument | undefined;
+
+    protected readonly langiumDocuments: LangiumDocuments;
     protected readonly astNodeLocator: AstNodeLocator;
     protected readonly nameProvider: NameProvider;
     protected readonly commentProvider: CommentProvider;
 
     constructor(services: LangiumServices) {
+        this.langiumDocuments = services.shared.workspace.LangiumDocuments;
         this.astNodeLocator = services.workspace.AstNodeLocator;
         this.nameProvider = services.references.NameProvider;
         this.commentProvider = services.documentation.CommentProvider;
@@ -105,51 +128,67 @@ export class DefaultJsonSerializer implements JsonSerializer {
         const defaultReplacer = (key: string, value: unknown) => this.replacer(key, value, options);
         const replacer = specificReplacer ? (key: string, value: unknown) => specificReplacer(key, value, defaultReplacer) : defaultReplacer;
 
-        return JSON.stringify(node, replacer, options?.space);
+        try {
+            this.currentDocument = getDocument(node);
+            return JSON.stringify(node, replacer, options?.space);
+        } finally {
+            this.currentDocument = undefined;
+        }
     }
 
-    deserialize(content: string): AstNode {
+    deserialize<T extends AstNode = AstNode>(content: string): T {
         const root = JSON.parse(content);
         this.linkNode(root, root);
         return root;
     }
 
-    protected replacer(key: string, value: unknown, { refText, sourceText, textRegions, comments }: JsonSerializeOptions = {}): unknown {
+    protected replacer(key: string, value: unknown, { refText, sourceText, textRegions, comments, uriConverter }: JsonSerializeOptions = {}): unknown {
         if (this.ignoreProperties.has(key)) {
             return undefined;
         } else if (isReference(value)) {
             const refValue = value.ref;
             const $refText = refText ? value.$refText : undefined;
             if (refValue) {
+                const targetDocument = getDocument(refValue);
+                let targetUri = '';
+                if (this.currentDocument && !UriUtils.equals(this.currentDocument.uri, targetDocument.uri)) {
+                    if (uriConverter) {
+                        targetUri = uriConverter(targetDocument.uri, value);
+                    } else {
+                        targetUri = targetDocument.uri.toString();
+                    }
+                }
+                const targetPath = this.astNodeLocator.getAstNodePath(refValue);
                 return {
-                    $refText,
-                    $ref: '#' + (refValue && this.astNodeLocator.getAstNodePath(refValue))
-                };
+                    $ref: `${targetUri}#${targetPath}`,
+                    $refText
+                } satisfies IntermediateReference;
             } else {
                 return {
-                    $refText,
-                    $error: value.error?.message ?? 'Could not resolve reference'
-                };
+                    $error: value.error?.message ?? 'Could not resolve reference',
+                    $refText
+                } satisfies IntermediateReference;
             }
-        } else {
+        } else if (isAstNode(value)) {
             let astNode: AstNodeWithTextRegion | undefined = undefined;
-            if (textRegions && isAstNode(value)) {
+            if (textRegions) {
                 astNode = this.addAstNodeRegionWithAssignmentsTo({ ...value });
                 if ((!key || value.$document) && astNode?.$textRegion) {
-                    try {
-                        astNode.$textRegion.documentURI = getDocument(value).uri.toString();
-                    } catch (e) { /* do nothing */ }
+                    // The document URI is added to the root node of the resulting JSON tree
+                    astNode.$textRegion.documentURI = this.currentDocument?.uri.toString();
                 }
             }
-            if (sourceText && !key && isAstNode(value)) {
+            if (sourceText && !key) {
                 astNode ??= { ...value };
                 astNode.$sourceText = value.$cstNode?.text;
             }
-            if (comments && isAstNode(value)) {
+            if (comments) {
                 astNode ??= { ...value };
                 (astNode as AstNodeWithComment).$comment = this.commentProvider.getComment(value);
             }
             return astNode ?? value;
+        } else {
+            return value;
         }
     }
 
@@ -194,31 +233,37 @@ export class DefaultJsonSerializer implements JsonSerializer {
                 this.linkNode(item as GenericAstNode, root, node, propertyName);
             }
         }
-        const mutable = node as Mutable<GenericAstNode>;
+        const mutable = node as Mutable<AstNode>;
         mutable.$container = container;
         mutable.$containerProperty = containerProperty;
         mutable.$containerIndex = containerIndex;
     }
 
     protected reviveReference(container: AstNode, property: string, root: AstNode, reference: IntermediateReference): Reference | undefined {
         let refText = reference.$refText;
+        let error = reference.$error;
         if (reference.$ref) {
             const ref = this.getRefNode(root, reference.$ref);
-            if (!refText) {
-                refText = this.nameProvider.getName(ref);
+            if (isAstNode(ref)) {
+                if (!refText) {
+                    refText = this.nameProvider.getName(ref);
+                }
+                return {
+                    $refText: refText ?? '',
+                    ref
+                };
+            } else {
+                error = ref;
             }
-            return {
-                $refText: refText ?? '',
-                ref
-            };
-        } else if (reference.$error) {
+        }
+        if (error) {
             const ref: Mutable<Reference> = {
                 $refText: refText ?? ''
             };
             ref.error = {
                 container,
                 property,
-                message: reference.$error,
+                message: error,
                 reference: ref
             };
             return ref;
@@ -227,7 +272,28 @@ export class DefaultJsonSerializer implements JsonSerializer {
         }
     }
 
-    protected getRefNode(root: AstNode, path: string): AstNode {
-        return this.astNodeLocator.getAstNode(root, path.substring(1))!;
+    protected getRefNode(root: AstNode, uri: string): AstNode | string {
+        try {
+            const fragmentIndex = uri.indexOf('#');
+            if (fragmentIndex === 0) {
+                const node = this.astNodeLocator.getAstNode(root, uri.substring(1));
+                if (!node) {
+                    return 'Could not resolve path: ' + uri;
+                }
+                return node;
+            }
+            const document = this.langiumDocuments.getOrCreateDocument(URI.parse(uri.substring(0, fragmentIndex)));
+            if (fragmentIndex < 0 || fragmentIndex === uri.length - 1) {
+                return document.parseResult.value;
+            }
+            const node = this.astNodeLocator.getAstNode(document.parseResult.value, uri.substring(fragmentIndex + 1));
+            if (!node) {
+                return 'Could not resolve URI: ' + uri;
+            }
+            return node;
+        } catch (err) {
+            return String(err);
+        }
     }
+
 }

packages/langium/src/workspace/file-system-provider.ts

@@ -38,11 +38,11 @@ export interface FileSystemProvider {
 export class EmptyFileSystemProvider implements FileSystemProvider {
 
     readFile(): Promise<string> {
-        throw new Error('Method not implemented.');
+        throw new Error('No file system is available.');
     }
 
     readFileSync(): string {
-        throw new Error('Method not implemented.');
+        throw new Error('No file system is available.');
     }
 
     async readDirectory(): Promise<FileSystemNode[]> {