Skip to content

Commit

Permalink
JsonSerializer: Support references to other documents (#1254)
Browse files Browse the repository at this point in the history
  • Loading branch information
@spoenemann
spoenemann authored Nov 9, 2023
1 parent bc6dabe commit 7066ac4
Show file tree
Hide file tree
Showing 6 changed files with 365 additions and 49 deletions.
13 changes: 11 additions & 2 deletions packages/langium-cli/src/generator/grammar-serializer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@
* terms of the MIT License, which is available in the project root.
******************************************************************************/

import type { Grammar, LangiumServices } from 'langium';
import type { URI } from 'vscode-uri';
import type { Grammar, LangiumServices, Reference } from 'langium';
import type { LangiumConfig } from '../package.js';
import { CompositeGeneratorNode, NL, normalizeEOL, toString } from 'langium';
import { generatedHeader } from './util.js';
Expand All @@ -29,9 +30,17 @@ export function serializeGrammar(services: LangiumServices, grammars: Grammar[],
if (grammar.name) {
const production = config.mode === 'production';
const delimiter = production ? "'" : '`';
const uriConverter = (uri: URI, ref: Reference) => {
// We expect the grammar to be self-contained after the transformations we've done before
throw new Error(`Unexpected reference to symbol '${ref.$refText}' in document: ${uri.toString()}`);
};
const serializedGrammar = services.serializer.JsonSerializer.serialize(grammar, {
space: production ? undefined : 2,
uriConverter
});
// The json serializer returns strings with \n line delimiter by default
// We need to translate these line endings to the OS specific line ending
const json = normalizeEOL(services.serializer.JsonSerializer.serialize(grammar, production ? undefined : { space: 2 })
const json = normalizeEOL(serializedGrammar
.replace(/\\/g, '\\\\')
.replace(new RegExp(delimiter, 'g'), '\\' + delimiter));
node.append(
Expand Down
4 changes: 2 additions & 2 deletions packages/langium-vscode/src/extension.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ function decorateText(document: vscode.TextDocument, decorations: vscode.Decorat
while ((match = regEx.exec(text))) {
const startIndex = textIndex + match.index;
const startPos = document.positionAt(startIndex + indentation);
const endPos = document.positionAt(startIndex + match[0].trimRight().length);
const endPos = document.positionAt(startIndex + match[0].trimEnd().length);
if (startPos.isBefore(endPos)) {
const range = new vscode.Range(startPos, endPos);
decorations.push({ range });
Expand All @@ -144,7 +144,7 @@ function decorateText(document: vscode.TextDocument, decorations: vscode.Decorat

// TODO(@@dd): find bug
function findIndentation(text: string): number {
const indents = text.split(/[\r?\n]/g).map(line => line.trimRight()).filter(line => line.length > 0).map(line => line.search(/\S|$/));
const indents = text.split(/[\r?\n]/g).map(line => line.trimEnd()).filter(line => line.length > 0).map(line => line.search(/\S|$/));
const min = indents.length === 0 ? 0 : Math.min(...indents); // min(...[]) = min() = Infinity
return Math.max(0, min);
}
Expand Down
4 changes: 2 additions & 2 deletions packages/langium/src/generator/template-string.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ function internalExpandToString(lineSep: string, staticParts: TemplateStringsArr
.split(NEWLINE_REGEXP)
.filter(l => l.trim() !== SNLE)
// whitespace-only lines are empty (preserving leading whitespace)
.map(l => l.replace(SNLE, '').trimRight());
.map(l => l.replace(SNLE, '').trimEnd());

// in order to nicely handle single line templates with the leading and trailing termintators (``) on separate lines, like
// expandToString`foo
Expand All @@ -65,7 +65,7 @@ function internalExpandToString(lineSep: string, staticParts: TemplateStringsArr
lines = containsLeadingLinebreak ? lines.slice(1) : lines;

// .. and drop the last linebreak if it's the last charactor or is followed by white space
const containsTrailingLinebreak = lines.length !== 0 && lines[lines.length-1].trimRight().length === 0;
const containsTrailingLinebreak = lines.length !== 0 && lines[lines.length-1].trimEnd().length === 0;
lines = containsTrailingLinebreak ? lines.slice(0, lines.length-1) : lines;

// finds the minimum indentation
Expand Down
158 changes: 117 additions & 41 deletions packages/langium/src/serializer/json-serializer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,33 @@ import type { LangiumServices } from '../services.js';
import type { AstNode, CstNode, GenericAstNode, Reference } from '../syntax-tree.js';
import type { Mutable } from '../utils/ast-util.js';
import type { AstNodeLocator } from '../workspace/ast-node-locator.js';
import type { DocumentSegment } from '../workspace/documents.js';
import type { DocumentSegment, LangiumDocument, LangiumDocuments } from '../workspace/documents.js';
import { URI } from 'vscode-uri';
import { isAstNode, isReference } from '../syntax-tree.js';
import { getDocument } from '../utils/ast-util.js';
import { findNodesForProperty } from '../utils/grammar-util.js';
import type { CommentProvider } from '../documentation/comment-provider.js';

export interface JsonSerializeOptions {
/** The space parameter for `JSON.stringify`, controlling whether and how to pretty-print the output. */
space?: string | number;
/** Whether to include the `$refText` property for references (the name used to identify the target node). */
refText?: boolean;
/** Whether to include the `$sourceText` property, which holds the full source text from which an AST node was parsed. */
sourceText?: boolean;
/** Whether to include the `$textRegion` property, which holds information to trace AST node properties to their respective source text regions. */
textRegions?: boolean;
/** Whether to include the `$comment` property, which holds comments according to the CommentProvider service. */
comments?: boolean;
/** The replacer parameter for `JSON.stringify`; the default replacer given as parameter should be used to apply basic replacements. */
replacer?: (key: string, value: unknown, defaultReplacer: (key: string, value: unknown) => unknown) => unknown
/** Used to convert and serialize URIs when the target of a cross-reference is in a different document. */
uriConverter?: (uri: URI, reference: Reference) => string
}

export interface JsonDeserializeOptions {
/** Used to parse and convert URIs when the target of a cross-reference is in a different document. */
uriConverter?: (uri: string) => URI
}

/**
Expand Down Expand Up @@ -50,7 +64,7 @@ export function isAstNodeWithComment(node: AstNode): node is AstNodeWithComment
*/
export interface AstNodeRegionWithAssignments extends DocumentSegment {
/**
* A record containing an entry for each assignd property of the AstNode.
* A record containing an entry for each assigned property of the AstNode.
* The key is equal to the property name and the value is an array of the property values'
* text regions, regardless of whether the property is a single value or list property.
*/
Expand All @@ -74,12 +88,18 @@ export interface JsonSerializer {
/**
* Deserialize (parse) a JSON `string` into an `AstNode`.
*/
deserialize(content: string): AstNode;
deserialize<T extends AstNode = AstNode>(content: string, options?: JsonDeserializeOptions): T;
}

/**
* A cross-reference in the serialized JSON representation of an AstNode.
*/
interface IntermediateReference {
$refText?: string
/** URI pointing to the target element. This is either `#${path}` if the target is in the same document, or `${documentURI}#${path}` otherwise. */
$ref?: string
/** The actual text used to look up the reference target in the surrounding scope. */
$refText?: string
/** If any problem occurred while resolving the reference, it is described by this property. */
$error?: string
}

Expand All @@ -89,67 +109,90 @@ function isIntermediateReference(obj: unknown): obj is IntermediateReference {

export class DefaultJsonSerializer implements JsonSerializer {

protected ignoreProperties = new Set(['$container', '$containerProperty', '$containerIndex', '$document', '$cstNode']);
/** The set of AstNode properties to be ignored by the serializer. */
ignoreProperties = new Set(['$container', '$containerProperty', '$containerIndex', '$document', '$cstNode']);

/** The document that is currently processed by the serializer; this is used by the replacer function. */
protected currentDocument: LangiumDocument | undefined;

protected readonly langiumDocuments: LangiumDocuments;
protected readonly astNodeLocator: AstNodeLocator;
protected readonly nameProvider: NameProvider;
protected readonly commentProvider: CommentProvider;

constructor(services: LangiumServices) {
this.langiumDocuments = services.shared.workspace.LangiumDocuments;
this.astNodeLocator = services.workspace.AstNodeLocator;
this.nameProvider = services.references.NameProvider;
this.commentProvider = services.documentation.CommentProvider;
}

serialize(node: AstNode, options?: JsonSerializeOptions): string {
serialize(node: AstNode, options: JsonSerializeOptions = {}): string {
const specificReplacer = options?.replacer;
const defaultReplacer = (key: string, value: unknown) => this.replacer(key, value, options);
const replacer = specificReplacer ? (key: string, value: unknown) => specificReplacer(key, value, defaultReplacer) : defaultReplacer;

return JSON.stringify(node, replacer, options?.space);
try {
this.currentDocument = getDocument(node);
return JSON.stringify(node, replacer, options?.space);
} finally {
this.currentDocument = undefined;
}
}

deserialize(content: string): AstNode {
deserialize<T extends AstNode = AstNode>(content: string, options: JsonDeserializeOptions = {}): T {
const root = JSON.parse(content);
this.linkNode(root, root);
this.linkNode(root, root, options);
return root;
}

protected replacer(key: string, value: unknown, { refText, sourceText, textRegions, comments }: JsonSerializeOptions = {}): unknown {
protected replacer(key: string, value: unknown, { refText, sourceText, textRegions, comments, uriConverter }: JsonSerializeOptions): unknown {
if (this.ignoreProperties.has(key)) {
return undefined;
} else if (isReference(value)) {
const refValue = value.ref;
const $refText = refText ? value.$refText : undefined;
if (refValue) {
const targetDocument = getDocument(refValue);
let targetUri = '';
if (this.currentDocument && this.currentDocument !== targetDocument) {
if (uriConverter) {
targetUri = uriConverter(targetDocument.uri, value);
} else {
targetUri = targetDocument.uri.toString();
}
}
const targetPath = this.astNodeLocator.getAstNodePath(refValue);
return {
$refText,
$ref: '#' + (refValue && this.astNodeLocator.getAstNodePath(refValue))
};
$ref: `${targetUri}#${targetPath}`,
$refText
} satisfies IntermediateReference;
} else {
return {
$refText,
$error: value.error?.message ?? 'Could not resolve reference'
};
$error: value.error?.message ?? 'Could not resolve reference',
$refText
} satisfies IntermediateReference;
}
} else {
} else if (isAstNode(value)) {
let astNode: AstNodeWithTextRegion | undefined = undefined;
if (textRegions && isAstNode(value)) {
if (textRegions) {
astNode = this.addAstNodeRegionWithAssignmentsTo({ ...value });
if ((!key || value.$document) && astNode?.$textRegion) {
try {
astNode.$textRegion.documentURI = getDocument(value).uri.toString();
} catch (e) { /* do nothing */ }
// The document URI is added to the root node of the resulting JSON tree
astNode.$textRegion.documentURI = this.currentDocument?.uri.toString();
}
}
if (sourceText && !key && isAstNode(value)) {
if (sourceText && !key) {
astNode ??= { ...value };
astNode.$sourceText = value.$cstNode?.text;
}
if (comments && isAstNode(value)) {
if (comments) {
astNode ??= { ...value };
(astNode as AstNodeWithComment).$comment = this.commentProvider.getComment(value);
}
return astNode ?? value;
} else {
return value;
}
}

Expand Down Expand Up @@ -177,48 +220,54 @@ export class DefaultJsonSerializer implements JsonSerializer {
return undefined;
}

protected linkNode(node: GenericAstNode, root: AstNode, container?: AstNode, containerProperty?: string, containerIndex?: number) {
protected linkNode(node: GenericAstNode, root: AstNode, options: JsonDeserializeOptions, container?: AstNode, containerProperty?: string, containerIndex?: number) {
for (const [propertyName, item] of Object.entries(node)) {
if (Array.isArray(item)) {
for (let index = 0; index < item.length; index++) {
const element = item[index];
if (isIntermediateReference(element)) {
item[index] = this.reviveReference(node, propertyName, root, element);
item[index] = this.reviveReference(node, propertyName, root, element, options);
} else if (isAstNode(element)) {
this.linkNode(element as GenericAstNode, root, node, propertyName, index);
this.linkNode(element as GenericAstNode, root, options, node, propertyName, index);
}
}
} else if (isIntermediateReference(item)) {
node[propertyName] = this.reviveReference(node, propertyName, root, item);
node[propertyName] = this.reviveReference(node, propertyName, root, item, options);
} else if (isAstNode(item)) {
this.linkNode(item as GenericAstNode, root, node, propertyName);
this.linkNode(item as GenericAstNode, root, options, node, propertyName);
}
}
const mutable = node as Mutable<GenericAstNode>;
const mutable = node as Mutable<AstNode>;
mutable.$container = container;
mutable.$containerProperty = containerProperty;
mutable.$containerIndex = containerIndex;
}

protected reviveReference(container: AstNode, property: string, root: AstNode, reference: IntermediateReference): Reference | undefined {
protected reviveReference(container: AstNode, property: string, root: AstNode, reference: IntermediateReference, options: JsonDeserializeOptions): Reference | undefined {
let refText = reference.$refText;
let error = reference.$error;
if (reference.$ref) {
const ref = this.getRefNode(root, reference.$ref);
if (!refText) {
refText = this.nameProvider.getName(ref);
const ref = this.getRefNode(root, reference.$ref, options.uriConverter);
if (isAstNode(ref)) {
if (!refText) {
refText = this.nameProvider.getName(ref);
}
return {
$refText: refText ?? '',
ref
};
} else {
error = ref;
}
return {
$refText: refText ?? '',
ref
};
} else if (reference.$error) {
}
if (error) {
const ref: Mutable<Reference> = {
$refText: refText ?? ''
};
ref.error = {
container,
property,
message: reference.$error,
message: error,
reference: ref
};
return ref;
Expand All @@ -227,7 +276,34 @@ export class DefaultJsonSerializer implements JsonSerializer {
}
}

protected getRefNode(root: AstNode, path: string): AstNode {
return this.astNodeLocator.getAstNode(root, path.substring(1))!;
protected getRefNode(root: AstNode, uri: string, uriConverter?: (uri: string) => URI): AstNode | string {
try {
const fragmentIndex = uri.indexOf('#');
if (fragmentIndex === 0) {
const node = this.astNodeLocator.getAstNode(root, uri.substring(1));
if (!node) {
return 'Could not resolve path: ' + uri;
}
return node;
}
if (fragmentIndex < 0) {
const documentUri = uriConverter ? uriConverter(uri) : URI.parse(uri);
const document = this.langiumDocuments.getOrCreateDocument(documentUri);
return document.parseResult.value;
}
const documentUri = uriConverter ? uriConverter(uri.substring(0, fragmentIndex)) : URI.parse(uri.substring(0, fragmentIndex));
const document = this.langiumDocuments.getOrCreateDocument(documentUri);
if (fragmentIndex === uri.length - 1) {
return document.parseResult.value;
}
const node = this.astNodeLocator.getAstNode(document.parseResult.value, uri.substring(fragmentIndex + 1));
if (!node) {
return 'Could not resolve URI: ' + uri;
}
return node;
} catch (err) {
return String(err);
}
}

}
4 changes: 2 additions & 2 deletions packages/langium/src/workspace/file-system-provider.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,11 @@ export interface FileSystemProvider {
export class EmptyFileSystemProvider implements FileSystemProvider {

readFile(): Promise<string> {
throw new Error('Method not implemented.');
throw new Error('No file system is available.');
}

readFileSync(): string {
throw new Error('Method not implemented.');
throw new Error('No file system is available.');
}

async readDirectory(): Promise<FileSystemNode[]> {
Expand Down
Loading

0 comments on commit 7066ac4

Please sign in to comment.