diff --git a/package-lock.json b/package-lock.json index b23645d25..9ef3403dc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,7 +32,7 @@ "require-relative": "^0.8.7", "roku-deploy": "^3.9.2", "serialize-error": "^7.0.1", - "source-map": "^0.7.3", + "source-map": "^0.7.4", "vscode-languageserver": "7.0.0", "vscode-languageserver-protocol": "3.16.0", "vscode-languageserver-textdocument": "^1.0.1", @@ -6138,9 +6138,9 @@ } }, "node_modules/source-map": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", - "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==", + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz", + "integrity": "sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==", "engines": { "node": ">= 8" } @@ -11659,9 +11659,9 @@ "dev": true }, "source-map": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", - "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==" + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz", + "integrity": "sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==" }, "source-map-support": { "version": "0.5.19", diff --git a/package.json b/package.json index b979f1898..3ff0c44ad 100644 --- a/package.json +++ b/package.json @@ -145,7 +145,7 @@ "require-relative": "^0.8.7", "roku-deploy": "^3.9.2", "serialize-error": "^7.0.1", - "source-map": "^0.7.3", + "source-map": "^0.7.4", "vscode-languageserver": "7.0.0", "vscode-languageserver-protocol": "3.16.0", "vscode-languageserver-textdocument": "^1.0.1", diff --git a/src/astUtils/creators.ts b/src/astUtils/creators.ts index 92ae752ad..6bded1c55 100644 --- a/src/astUtils/creators.ts +++ b/src/astUtils/creators.ts @@ -89,7 +89,8 @@ export function createToken(kind: T, text?: string, range = text: text ?? tokenDefaults[kind as string] ?? kind.toString().toLowerCase(), isReserved: !text || text === kind.toString(), range: range, - leadingWhitespace: '' + leadingWhitespace: '', + leadingTrivia: [] }; } @@ -99,7 +100,8 @@ export function createIdentifier(name: string, range?: Range): Identifier { text: name, isReserved: false, range: range, - leadingWhitespace: '' + leadingWhitespace: '', + leadingTrivia: [] }; } @@ -169,7 +171,8 @@ export function createCall(callee: Expression, args?: Expression[]) { callee, createToken(TokenKind.LeftParen, '('), createToken(TokenKind.RightParen, ')'), - args || [] + args || [], + [] ); } diff --git a/src/astUtils/reflection.spec.ts b/src/astUtils/reflection.spec.ts index 8efbc1b95..66370dd23 100644 --- a/src/astUtils/reflection.spec.ts +++ b/src/astUtils/reflection.spec.ts @@ -47,8 +47,8 @@ describe('reflection', () => { const fors = new ForStatement(token, assignment, token, expr, block, token, token, expr); const foreach = new ForEachStatement({ forEach: token, in: token, endFor: token }, token, expr, block); const whiles = new WhileStatement({ while: token, endWhile: token }, expr, block); - const dottedSet = new DottedSetStatement(expr, ident, expr); - const indexedSet = new IndexedSetStatement(expr, expr, expr, token, token); + const dottedSet = new DottedSetStatement(expr, ident, expr, token, token); + const indexedSet = new IndexedSetStatement(expr, expr, expr, token, token, token); const library = new LibraryStatement({ library: token, filePath: token }); const namespace = new NamespaceStatement(token, new NamespacedVariableNameExpression(createVariableExpression('a', range)), body, token); const cls = new ClassStatement(token, ident, [], token); @@ -193,11 +193,12 @@ describe('reflection', () => { range: range, isReserved: false, charCode: 0, - leadingWhitespace: '' + leadingWhitespace: '', + leadingTrivia: [] }; const nsVar = new NamespacedVariableNameExpression(createVariableExpression('a', range)); const binary = new BinaryExpression(expr, token, expr); - const call = new CallExpression(expr, token, token, []); + const call = new CallExpression(expr, token, token, [], []); const fun = new FunctionExpression([], block, token, token, token, token); const dottedGet = new DottedGetExpression(expr, ident, token); const xmlAttrGet = new XmlAttributeGetExpression(expr, ident, token); @@ -205,16 +206,16 @@ describe('reflection', () => { const grouping = new GroupingExpression({ left: token, right: token }, expr); const literal = createStringLiteral('test'); const escapedCarCode = new EscapedCharCodeLiteralExpression(charCode); - const arrayLit = new ArrayLiteralExpression([], token, token); + const arrayLit = new ArrayLiteralExpression([], token, token, []); const aaLit = new AALiteralExpression([], token, token); const unary = new UnaryExpression(token, expr); const variable = new VariableExpression(ident); const sourceLit = new SourceLiteralExpression(token); const newx = new NewExpression(token, call); - const callfunc = new CallfuncExpression(expr, token, ident, token, [], token); + const callfunc = new CallfuncExpression(expr, token, ident, token, [], token, []); const tplQuasi = new TemplateStringQuasiExpression([expr]); - const tplString = new TemplateStringExpression(token, [tplQuasi], [], token); - const taggedTpl = new TaggedTemplateStringExpression(ident, token, [tplQuasi], [], token); + const tplString = new TemplateStringExpression(token, [tplQuasi], [], token, [], []); + const taggedTpl = new TaggedTemplateStringExpression(ident, token, [tplQuasi], [], token, [], []); const annotation = new AnnotationExpression(token, token); it('isExpression', () => { diff --git a/src/astUtils/reflection.ts b/src/astUtils/reflection.ts index b353b587e..b9480cc96 100644 --- a/src/astUtils/reflection.ts +++ b/src/astUtils/reflection.ts @@ -20,6 +20,7 @@ import { DynamicType } from '../types/DynamicType'; import type { InterfaceType } from '../types/InterfaceType'; import type { ObjectType } from '../types/ObjectType'; import type { AstNode, Expression, Statement } from '../parser/AstNode'; +import type { Token } from '../lexer/Token'; // File reflection @@ -183,9 +184,9 @@ export function isThrowStatement(element: AstNode | undefined): element is Throw * this will work for StringLiteralExpression -> Expression, * but will not work CustomStringLiteralExpression -> StringLiteralExpression -> Expression */ -export function isExpression(element: AstNode | undefined): element is Expression { +export function isExpression(element: AstNode | Token | undefined): element is Expression { // eslint-disable-next-line no-bitwise - return !!(element && element.visitMode & InternalWalkMode.visitExpressions); + return !!(element && (element as any).visitMode & InternalWalkMode.visitExpressions); } export function isBinaryExpression(element: AstNode | undefined): element is BinaryExpression { diff --git a/src/lexer/Lexer.spec.ts b/src/lexer/Lexer.spec.ts index 721f0925c..cc187e04c 100644 --- a/src/lexer/Lexer.spec.ts +++ b/src/lexer/Lexer.spec.ts @@ -1,8 +1,8 @@ /* eslint no-template-curly-in-string: 0 */ import { expect } from '../chai-config.spec'; - import { TokenKind } from './TokenKind'; -import { Lexer } from './Lexer'; +import { Lexer, triviaKinds } from './Lexer'; +import type { Token } from './Token'; import { isToken } from './Token'; import { rangeToArray } from '../parser/Parser.spec'; import { Range } from 'vscode-languageserver'; @@ -1377,6 +1377,63 @@ describe('lexer', () => { TokenKind.Eof ]); }); + + describe('trivia', () => { + function stringify(tokens: Token[]) { + return tokens + //exclude the explicit triva tokens since they'll be included in the leading/trailing arrays + .filter(x => !triviaKinds.includes(x.kind)) + .flatMap(x => [...x.leadingTrivia, x]) + .map(x => x.text) + .join(''); + } + + it('combining token text and trivia can reproduce full input', () => { + const input = ` + function test( ) + 'comment + print alpha ' blabla + end function 'trailing + 'trailing2 + `; + expect( + stringify( + Lexer.scan(input).tokens + ) + ).to.eql(input); + }); + + function expectTrivia(text: string, expected: Array<{ text: string; leadingTrivia?: string[]; trailingTrivia?: string[] }>) { + const tokens = Lexer.scan(text).tokens.filter(x => !triviaKinds.includes(x.kind)); + expect( + tokens.map(x => { + return { + text: x.text, + leadingTrivia: x.leadingTrivia.map(x => x.text) + }; + }) + ).to.eql( + expected.map(x => ({ + leadingTrivia: [], + ...x + })) + ); + } + + it('associates trailing items on same line with the preceeding token', () => { + expectTrivia( + `'leading\n` + + `alpha = true 'trueComment\n` + + `'eof` + , [ + { leadingTrivia: [`'leading`, `\n`], text: `alpha` }, + { leadingTrivia: [` `], text: `=` }, + { leadingTrivia: [` `], text: `true` }, + //EOF + { leadingTrivia: [` `, `'trueComment`, `\n`, `'eof`], text: `` } + ]); + }); + }); }); function expectKinds(text: string, tokenKinds: TokenKind[]) { diff --git a/src/lexer/Lexer.ts b/src/lexer/Lexer.ts index 0493d8b94..246fddc87 100644 --- a/src/lexer/Lexer.ts +++ b/src/lexer/Lexer.ts @@ -6,6 +6,13 @@ import type { Range, Diagnostic } from 'vscode-languageserver'; import { DiagnosticMessages } from '../DiagnosticMessages'; import util from '../util'; +export const triviaKinds: ReadonlyArray = [ + TokenKind.Newline, + TokenKind.Whitespace, + TokenKind.Comment, + TokenKind.Colon +]; + export class Lexer { /** * The zero-indexed position at which the token under consideration begins. @@ -103,12 +110,22 @@ export class Lexer { isReserved: false, text: '', range: util.createRange(this.lineBegin, this.columnBegin, this.lineEnd, this.columnEnd + 1), - leadingWhitespace: this.leadingWhitespace + leadingWhitespace: this.leadingWhitespace, + leadingTrivia: this.leadingTrivia }); this.leadingWhitespace = ''; return this; } + private leadingTrivia: Token[] = []; + + /** + * Pushes a token into the leadingTrivia list + */ + private pushTrivia(token: Token) { + this.leadingTrivia.push(token); + } + /** * Fill in missing/invalid options with defaults */ @@ -1031,6 +1048,13 @@ export class Lexer { return false; } + /** + * Determine if this token is a trivia token + */ + private isTrivia(token: Token) { + return triviaKinds.includes(token.kind); + } + /** * Creates a `Token` and adds it to the `tokens` array. * @param kind the type of token to produce. @@ -1042,8 +1066,15 @@ export class Lexer { text: text, isReserved: ReservedWords.has(text.toLowerCase()), range: this.rangeOf(), - leadingWhitespace: this.leadingWhitespace + leadingWhitespace: this.leadingWhitespace, + leadingTrivia: [] }; + if (this.isTrivia(token)) { + this.pushTrivia(token); + } else { + token.leadingTrivia.push(...this.leadingTrivia); + this.leadingTrivia = []; + } this.leadingWhitespace = ''; this.tokens.push(token); this.sync(); diff --git a/src/lexer/Token.ts b/src/lexer/Token.ts index 42e814123..451c8b6ec 100644 --- a/src/lexer/Token.ts +++ b/src/lexer/Token.ts @@ -17,6 +17,10 @@ export interface Token { * Any leading whitespace found prior to this token. Excludes newline characters. */ leadingWhitespace?: string; + /** + * Any tokens starting on the next line of the previous token, up to the start of this token + */ + leadingTrivia: Token[]; } /** diff --git a/src/parser/AstNode.spec.ts b/src/parser/AstNode.spec.ts index 329fa5ca0..ba962382e 100644 --- a/src/parser/AstNode.spec.ts +++ b/src/parser/AstNode.spec.ts @@ -6,6 +6,7 @@ import { expect } from '../chai-config.spec'; import type { DottedGetExpression } from './Expression'; import { expectZeroDiagnostics } from '../testHelpers.spec'; import { tempDir, rootDir, stagingDir } from '../testHelpers.spec'; +import { Parser } from './Parser'; describe('Program', () => { let program: Program; @@ -42,4 +43,45 @@ describe('Program', () => { }); }); }); + + describe('toString', () => { + function testToString(text: string) { + expect( + Parser.parse(text).ast.toString() + ).to.eql( + text + ); + } + it('retains full fidelity', () => { + testToString(` + thing = true + + if true + thing = true + end if + + if true + thing = true + else + thing = true + end if + + if true + thing = true + else if true + thing = true + else + thing = true + end if + + for i = 0 to 10 step 1 + print true,false;3 + end for + + for each item in thing + print 1 + end for + `); + }); + }); }); diff --git a/src/parser/AstNode.ts b/src/parser/AstNode.ts index 6de713722..81326ed6a 100644 --- a/src/parser/AstNode.ts +++ b/src/parser/AstNode.ts @@ -8,6 +8,8 @@ import type { BrsTranspileState } from './BrsTranspileState'; import type { TranspileResult } from '../interfaces'; import type { AnnotationExpression } from './Expression'; import util from '../util'; +import type { SourceNode } from 'source-map'; +import { TranspileState } from './TranspileState'; /** * A BrightScript AST node @@ -104,6 +106,20 @@ export abstract class AstNode { walkMode: WalkMode.visitAllRecursive }); } + + /** + * Return the string value of this AstNode + */ + public toString() { + return this + .toSourceNode(new TranspileState('', {})) + .toString(); + } + + /** + * Generate a SourceNode that represents the stringified value of this node (used to generate sourcemaps and transpile the code + */ + public abstract toSourceNode(state: TranspileState): SourceNode; } export abstract class Statement extends AstNode { diff --git a/src/parser/Expression.ts b/src/parser/Expression.ts index 25d80211a..019f783bf 100644 --- a/src/parser/Expression.ts +++ b/src/parser/Expression.ts @@ -10,7 +10,7 @@ import * as fileUrl from 'file-url'; import type { WalkOptions, WalkVisitor } from '../astUtils/visitors'; import { createVisitor, WalkMode } from '../astUtils/visitors'; import { walk, InternalWalkMode, walkArray } from '../astUtils/visitors'; -import { isAALiteralExpression, isArrayLiteralExpression, isCallExpression, isCallfuncExpression, isCommentStatement, isDottedGetExpression, isEscapedCharCodeLiteralExpression, isFunctionExpression, isIntegerType, isLiteralBoolean, isLiteralExpression, isLiteralNumber, isLiteralString, isLongIntegerType, isNamespaceStatement, isStringType, isUnaryExpression, isVariableExpression } from '../astUtils/reflection'; +import { isAALiteralExpression, isArrayLiteralExpression, isCallExpression, isCallfuncExpression, isCommentStatement, isDottedGetExpression, isEscapedCharCodeLiteralExpression, isFunctionExpression, isFunctionStatement, isIntegerType, isLiteralBoolean, isLiteralExpression, isLiteralNumber, isLiteralString, isLongIntegerType, isNamespaceStatement, isStringType, isUnaryExpression, isVariableExpression } from '../astUtils/reflection'; import type { TranspileResult, TypedefProvider } from '../interfaces'; import { VoidType } from '../types/VoidType'; import { DynamicType } from '../types/DynamicType'; @@ -18,6 +18,8 @@ import type { BscType } from '../types/BscType'; import { FunctionType } from '../types/FunctionType'; import { Expression } from './AstNode'; import { SymbolTable } from '../SymbolTable'; +import type { TranspileState } from './TranspileState'; +import { SourceNode } from 'source-map'; export type ExpressionVisitor = (expression: Expression, parent: Expression) => void; @@ -43,6 +45,14 @@ export class BinaryExpression extends Expression { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + this.left?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.operator), + this.right?.toSourceNode(state) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'left', visitor, options); @@ -61,7 +71,8 @@ export class CallExpression extends Expression { */ readonly openingParen: Token, readonly closingParen: Token, - readonly args: Expression[] + readonly args: Expression[], + readonly argCommas: Token[] ) { super(); this.range = util.createBoundingRange(this.callee, this.openingParen, ...args, this.closingParen); @@ -106,6 +117,18 @@ export class CallExpression extends Expression { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + this.callee?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.openingParen), + ...this.args.map((x, i) => ([ + x.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.argCommas[i]) + ])).flat(), + state.tokenToSourceNodeWithTrivia(this.closingParen) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'callee', visitor, options); @@ -123,7 +146,8 @@ export class FunctionExpression extends Expression implements TypedefProvider { readonly leftParen: Token, readonly rightParen: Token, readonly asToken?: Token, - readonly returnTypeToken?: Token + readonly returnTypeToken?: Token, + readonly paramCommas?: Token[] ) { super(); if (this.returnTypeToken) { @@ -257,6 +281,24 @@ export class FunctionExpression extends Expression implements TypedefProvider { return results; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.functionType), + //include the name (if we have a parent FunctionStatement) + isFunctionStatement(this.parent) ? state.tokenToSourceNodeWithTrivia(this.parent.name) : undefined, + state.tokenToSourceNodeWithTrivia(this.leftParen), + ...this.parameters?.map((x, i) => ([ + x.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.paramCommas?.[i]) + ])).flat() ?? [], + state.tokenToSourceNodeWithTrivia(this.rightParen), + state.tokenToSourceNodeWithTrivia(this.asToken), + state.tokenToSourceNodeWithTrivia(this.returnTypeToken), + this.body?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.end) + ); + } + getTypedef(state: BrsTranspileState, name?: Identifier) { return this.transpile(state, name, false); } @@ -287,7 +329,8 @@ export class FunctionParameterExpression extends Expression { public name: Identifier, public typeToken?: Token, public defaultValue?: Expression, - public asToken?: Token + public asToken?: Token, + readonly equalsToken?: Token ) { super(); if (typeToken) { @@ -327,6 +370,16 @@ export class FunctionParameterExpression extends Expression { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.name), + state.tokenToSourceNodeWithTrivia(this.equalsToken), + this.defaultValue?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.asToken), + state.tokenToSourceNodeWithTrivia(this.typeToken) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { // eslint-disable-next-line no-bitwise if (this.defaultValue && options.walkMode & InternalWalkMode.walkExpressions) { @@ -351,6 +404,10 @@ export class NamespacedVariableNameExpression extends Expression { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return this.expression?.toSourceNode(state); + } + public getNameParts() { let parts = [] as string[]; if (isVariableExpression(this.expression)) { @@ -412,12 +469,19 @@ export class DottedGetExpression extends Expression { } } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + this.obj?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.dot), + state.tokenToSourceNodeWithTrivia(this.name) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'obj', visitor, options); } } - } export class XmlAttributeGetExpression extends Expression { @@ -443,6 +507,14 @@ export class XmlAttributeGetExpression extends Expression { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + this.obj?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.at), + state.tokenToSourceNodeWithTrivia(this.name) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'obj', visitor, options); @@ -477,6 +549,16 @@ export class IndexedGetExpression extends Expression { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + this.obj?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.questionDotToken), + state.tokenToSourceNodeWithTrivia(this.openingSquare), + this.index?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.closingSquare) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'obj', visitor, options); @@ -507,6 +589,14 @@ export class GroupingExpression extends Expression { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.left), + this.expression?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.tokens.right) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'expression', visitor, options); @@ -552,6 +642,12 @@ export class LiteralExpression extends Expression { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return new SourceNode(0, 0, null, [ + state.tokenToSourceNodeWithTrivia(this.token) + ]); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -576,6 +672,10 @@ export class EscapedCharCodeLiteralExpression extends Expression { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.tokenToSourceNodeWithTrivia(this.token); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -586,7 +686,12 @@ export class ArrayLiteralExpression extends Expression { readonly elements: Array, readonly open: Token, readonly close: Token, - readonly hasSpread = false + /** + * An array of commas used to separate elements. + * Since commas are optional, but we need indexes to be aligned, + * there should be an entry in here for every element, even if `undefined` + */ + readonly commas: Array ) { super(); this.range = util.createBoundingRange(this.open, ...this.elements, this.close); @@ -642,6 +747,17 @@ export class ArrayLiteralExpression extends Expression { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.open), + ...this.elements?.map((x, i) => ([ + x?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.commas[i]) + ])).flat() ?? [], + state.tokenToSourceNodeWithTrivia(this.close) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walkArray(this.elements, visitor, options, this); @@ -668,6 +784,15 @@ export class AAMemberExpression extends Expression { return []; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.keyToken), + //colon tokens are technically trivia, so we don't need to include that here + this.value?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.commaToken) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { walk(this, 'value', visitor, options); } @@ -756,6 +881,14 @@ export class AALiteralExpression extends Expression { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.open), + state.arrayToSourceNodeWithTrivia(this.elements, isCommentStatement), + state.tokenToSourceNodeWithTrivia(this.close) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walkArray(this.elements, visitor, options, this); @@ -782,6 +915,13 @@ export class UnaryExpression extends Expression { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.operator), + this.right?.toSourceNode(state) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'right', visitor, options); @@ -825,6 +965,10 @@ export class VariableExpression extends Expression { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.tokenToSourceNodeWithTrivia(this.name); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -901,6 +1045,10 @@ export class SourceLiteralExpression extends Expression { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.tokenToSourceNodeWithTrivia(this.token); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -942,6 +1090,13 @@ export class NewExpression extends Expression { return this.call.transpile(state, cls?.getName(ParseMode.BrightScript)); } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.newKeyword), + this.call?.toSourceNode(state) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'call', visitor, options); @@ -956,7 +1111,8 @@ export class CallfuncExpression extends Expression { readonly methodName: Identifier, readonly openingParen: Token, readonly args: Expression[], - readonly closingParen: Token + readonly closingParen: Token, + readonly argCommas: Token[] ) { super(); this.range = util.createRangeFromPositions( @@ -1005,6 +1161,20 @@ export class CallfuncExpression extends Expression { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + this.callee?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.operator), + state.tokenToSourceNodeWithTrivia(this.methodName), + state.tokenToSourceNodeWithTrivia(this.openingParen), + ...this.args.map((x, i) => ([ + x.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.argCommas[i]) + ]))?.flat() ?? [], + state.tokenToSourceNodeWithTrivia(this.closingParen) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'callee', visitor, options); @@ -1047,6 +1217,12 @@ export class TemplateStringQuasiExpression extends Expression { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + ...this.expressions.map(x => x.toSourceNode(state)) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walkArray(this.expressions, visitor, options, this); @@ -1059,7 +1235,9 @@ export class TemplateStringExpression extends Expression { readonly openingBacktick: Token, readonly quasis: TemplateStringQuasiExpression[], readonly expressions: Expression[], - readonly closingBacktick: Token + readonly closingBacktick: Token, + readonly expressionBeginTokens: Token[], + readonly expressionEndTokens: Token[] ) { super(); this.range = util.createRangeFromPositions( @@ -1121,6 +1299,21 @@ export class TemplateStringExpression extends Expression { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.openingBacktick), + ...this.quasis?.map((x, i) => ([ + this.quasis[i]?.toSourceNode(state), + ...(this.expressions?.[i] ? [ + state.tokenToSourceNodeWithTrivia(this.expressionBeginTokens[i]), + this.expressions[i]?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.expressionEndTokens[i]) + ] : []) + ])).flat() ?? [], + state.tokenToSourceNodeWithTrivia(this.closingBacktick) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { //walk the quasis and expressions in left-to-right order @@ -1142,7 +1335,9 @@ export class TaggedTemplateStringExpression extends Expression { readonly openingBacktick: Token, readonly quasis: TemplateStringQuasiExpression[], readonly expressions: Expression[], - readonly closingBacktick: Token + readonly closingBacktick: Token, + readonly expressionBeginTokens: Token[], + readonly expressionEndTokens: Token[] ) { super(); this.range = util.createRangeFromPositions( @@ -1195,6 +1390,22 @@ export class TaggedTemplateStringExpression extends Expression { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tagName), + state.tokenToSourceNodeWithTrivia(this.openingBacktick), + ...this.quasis?.map((x, i) => ([ + this.quasis[i]?.toSourceNode(state), + ...(this.expressions?.[i] ? [ + state.tokenToSourceNodeWithTrivia(this.expressionBeginTokens[i]), + this.expressions[i]?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.expressionEndTokens[i]) + ] : []) + ])).flat() ?? [], + state.tokenToSourceNodeWithTrivia(this.closingBacktick) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { //walk the quasis and expressions in left-to-right order @@ -1242,6 +1453,13 @@ export class AnnotationExpression extends Expression { return []; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.atToken), + this.call.toSourceNode(state) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -1330,6 +1548,17 @@ export class TernaryExpression extends Expression { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + this.test?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.questionMarkToken), + this.consequent?.toSourceNode(state), + // don't add the colon because colons are included in leadingTrivia + // state.tokenToSourceNodeWithTrivia(this.colonToken), + this.alternate?.toSourceNode(state) + ); + } + public walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'test', visitor, options); @@ -1413,6 +1642,14 @@ export class NullCoalescingExpression extends Expression { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + this.consequent?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.questionQuestionToken), + this.alternate?.toSourceNode(state) + ); + } + public walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'consequent', visitor, options); @@ -1459,6 +1696,12 @@ export class RegexLiteralExpression extends Expression { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.regexLiteral) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } diff --git a/src/parser/Parser.ts b/src/parser/Parser.ts index c44d0fd2f..e8b6f2a89 100644 --- a/src/parser/Parser.ts +++ b/src/parser/Parser.ts @@ -230,6 +230,8 @@ export class Parser { this.pendingAnnotations = []; this.ast = this.body(); + //assign the eofToken to body + this.ast.eofToken = this.tokens[this.tokens.length - 1]; //now that we've built the AST, link every node to its parent this.ast.link(); @@ -782,7 +784,8 @@ export class Parser { start: this.peek().range.start, end: this.peek().range.start }, - leadingWhitespace: '' + leadingWhitespace: '', + leadingTrivia: [] }; } let isSub = functionType?.kind === TokenKind.Sub; @@ -828,8 +831,9 @@ export class Parser { let params = [] as FunctionParameterExpression[]; let asToken: Token; let typeToken: Token; + const paramCommas: Token[] = []; if (!this.check(TokenKind.RightParen)) { - do { + while (true) { if (params.length >= CallExpression.MaximumArguments) { this.diagnostics.push({ ...DiagnosticMessages.tooManyCallableParameters(params.length, CallExpression.MaximumArguments), @@ -838,7 +842,12 @@ export class Parser { } params.push(this.functionParameter()); - } while (this.match(TokenKind.Comma)); + if (this.check(TokenKind.Comma)) { + paramCommas.push(this.advance()); + } else { + break; + } + } } let rightParen = this.advance(); @@ -876,7 +885,8 @@ export class Parser { leftParen, rightParen, asToken, - typeToken + typeToken, + paramCommas ); // add the function to the relevant symbol tables @@ -891,7 +901,9 @@ export class Parser { try { //support ending the function with `end sub` OR `end function` func.body = this.block(); - func.body.symbolTable = new SymbolTable(`Block: Function '${name?.text ?? ''}'`, () => func.getSymbolTable()); + if (func.body) { + func.body.symbolTable = new SymbolTable(`Block: Function '${name?.text ?? ''}'`, () => func.getSymbolTable()); + } } finally { } if (!func.body) { @@ -948,9 +960,10 @@ export class Parser { let typeToken: Token | undefined; let defaultValue; - + let equalsToken: Token; // parse argument default value if (this.match(TokenKind.Equal)) { + equalsToken = this.previous(); // it seems any expression is allowed here -- including ones that operate on other arguments! defaultValue = this.expression(); } @@ -972,7 +985,8 @@ export class Parser { name, typeToken, defaultValue, - asToken + asToken, + equalsToken ); } @@ -1531,6 +1545,10 @@ export class Parser { let openingBacktick = this.peek(); this.advance(); let currentQuasiExpressionParts = []; + + let expressionBeginTokens: Token[] = []; + let expressionEndTokens: Token[] = []; + while (!this.isAtEnd() && !this.check(TokenKind.BackTick)) { let next = this.peek(); if (next.kind === TokenKind.TemplateStringQuasi) { @@ -1552,13 +1570,17 @@ export class Parser { currentQuasiExpressionParts = []; if (next.kind === TokenKind.TemplateStringExpressionBegin) { - this.advance(); + expressionBeginTokens.push( + this.advance() + ); } //now keep this expression expressions.push(this.expression()); if (!this.isAtEnd() && this.check(TokenKind.TemplateStringExpressionEnd)) { //TODO is it an error if this is not present? - this.advance(); + expressionEndTokens.push( + this.advance() + ); } else { this.diagnostics.push({ ...DiagnosticMessages.unterminatedTemplateExpression(), @@ -1585,9 +1607,9 @@ export class Parser { } else { let closingBacktick = this.advance(); if (isTagged) { - return new TaggedTemplateStringExpression(tagName, openingBacktick, quasis, expressions, closingBacktick); + return new TaggedTemplateStringExpression(tagName, openingBacktick, quasis, expressions, closingBacktick, expressionBeginTokens, expressionEndTokens); } else { - return new TemplateStringExpression(openingBacktick, quasis, expressions, closingBacktick); + return new TemplateStringExpression(openingBacktick, quasis, expressions, closingBacktick, expressionBeginTokens, expressionEndTokens); } } } @@ -2000,7 +2022,8 @@ export class Parser { ? right : new BinaryExpression(left, operator, right), left.openingSquare, - left.closingSquare + left.closingSquare, + operator ); } else if (isDottedGetExpression(left)) { return new DottedSetStatement( @@ -2008,7 +2031,9 @@ export class Parser { left.name, operator.kind === TokenKind.Equal ? right - : new BinaryExpression(left, operator, right) + : new BinaryExpression(left, operator, right), + left.dot, + operator ); } } @@ -2405,7 +2430,7 @@ export class Parser { let openParen = this.consume(DiagnosticMessages.expectedOpenParenToFollowCallfuncIdentifier(), TokenKind.LeftParen); let call = this.finishCall(openParen, callee, false); - return new CallfuncExpression(callee, operator, methodName as Identifier, openParen, call.args, call.closingParen); + return new CallfuncExpression(callee, operator, methodName as Identifier, openParen, call.args, call.closingParen, call.argCommas); } private call(): Expression { @@ -2481,9 +2506,10 @@ export class Parser { private finishCall(openingParen: Token, callee: Expression, addToCallExpressionList = true) { let args = [] as Expression[]; while (this.match(TokenKind.Newline)) { } + const commas: Token[] = []; if (!this.check(TokenKind.RightParen)) { - do { + while (true) { while (this.match(TokenKind.Newline)) { } if (args.length >= CallExpression.MaximumArguments) { @@ -2500,7 +2526,12 @@ export class Parser { // we were unable to get an expression, so don't continue break; } - } while (this.match(TokenKind.Comma)); + if (this.check(TokenKind.Comma)) { + commas.push(this.advance()); + } else { + break; + } + } } while (this.match(TokenKind.Newline)) { } @@ -2510,7 +2541,7 @@ export class Parser { TokenKind.RightParen ); - let expression = new CallExpression(callee, openingParen, closingParen, args); + let expression = new CallExpression(callee, openingParen, closingParen, args, commas); if (addToCallExpressionList) { this.callExpressions.push(expression); } @@ -2524,6 +2555,7 @@ export class Parser { */ private typeToken(): Token { let typeToken: Token; + const leadingTrivia = this.peek()?.leadingTrivia; if (this.checkAny(...DeclarableTypes)) { // Token is a built in type @@ -2541,6 +2573,9 @@ export class Parser { // just get whatever's next typeToken = this.advance(); } + if (typeToken) { + typeToken.leadingTrivia = leadingTrivia; + } return typeToken; } @@ -2621,6 +2656,7 @@ export class Parser { private arrayLiteral() { let elements: Array = []; + let commas: Array = []; let openingSquare = this.previous(); //add any comment found right after the opening square @@ -2637,19 +2673,22 @@ export class Parser { elements.push(this.expression()); while (this.matchAny(TokenKind.Comma, TokenKind.Newline, TokenKind.Comment)) { + const previous = this.previous(); if (this.checkPrevious(TokenKind.Comment) || this.check(TokenKind.Comment)) { let comment = this.check(TokenKind.Comment) ? this.advance() : this.previous(); elements.push(new CommentStatement([comment])); } - while (this.match(TokenKind.Newline)) { - - } + while (this.match(TokenKind.Newline)) { } if (this.check(TokenKind.RightSquareBracket)) { break; } elements.push(this.expression()); + commas.push( + previous?.kind === TokenKind.Comma ? previous : undefined + ); + } } catch (error: any) { this.rethrowNonDiagnosticError(error); @@ -2664,7 +2703,7 @@ export class Parser { } //this.consume("Expected newline or ':' after array literal", TokenKind.Newline, TokenKind.Colon, TokenKind.Eof); - return new ArrayLiteralExpression(elements, openingSquare, closingSquare); + return new ArrayLiteralExpression(elements, openingSquare, closingSquare, commas); } private aaLiteral() { diff --git a/src/parser/Statement.ts b/src/parser/Statement.ts index 558f641b2..6d4b96b51 100644 --- a/src/parser/Statement.ts +++ b/src/parser/Statement.ts @@ -15,7 +15,7 @@ import type { TranspileResult, TypedefProvider } from '../interfaces'; import { createInvalidLiteral, createMethodStatement, createToken, interpolatedRange } from '../astUtils/creators'; import { DynamicType } from '../types/DynamicType'; import type { BscType } from '../types/BscType'; -import type { SourceNode } from 'source-map'; +import { SourceNode } from 'source-map'; import type { TranspileState } from './TranspileState'; import { SymbolTable } from '../SymbolTable'; import type { Expression } from './AstNode'; @@ -34,6 +34,10 @@ export class EmptyStatement extends Statement { transpile(state: BrsTranspileState) { return []; } + + toSourceNode() { + return new SourceNode(); + } walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -44,7 +48,11 @@ export class EmptyStatement extends Statement { */ export class Body extends Statement implements TypedefProvider { constructor( - public statements: Statement[] = [] + public statements: Statement[] = [], + /** + * If this is the top-level program body, it will have the EOF token attached + */ + public eofToken?: Token ) { super(); } @@ -91,6 +99,13 @@ export class Body extends Statement implements TypedefProvider { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.arrayToSourceNodeWithTrivia(this.statements, isCommentStatement), + this.eofToken ? state.tokenToSourceNodeWithTrivia(this.eofToken) : '' + ); + } + getTypedef(state: BrsTranspileState) { let result = []; for (const statement of this.statements) { @@ -148,6 +163,14 @@ export class AssignmentStatement extends Statement { } } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.name), + state.tokenToSourceNodeWithTrivia(this.equals), + this.value.toSourceNode(state) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'value', visitor, options); @@ -204,6 +227,10 @@ export class Block extends Statement { return results; } + public toSourceNode(state: TranspileState): SourceNode { + return state.arrayToSourceNodeWithTrivia(this.statements, isCommentStatement); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkStatements) { walkArray(this.statements, visitor, options, this); @@ -225,6 +252,10 @@ export class ExpressionStatement extends Statement { return this.expression.transpile(state); } + public toSourceNode(state: TranspileState): SourceNode { + return this.expression.toSourceNode(state); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'expression', visitor, options); @@ -275,6 +306,12 @@ export class CommentStatement extends Statement implements Expression, TypedefPr return this.transpile(state as BrsTranspileState); } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + ...this.comments.map(x => state.tokenToSourceNodeWithTrivia(x)) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -298,6 +335,10 @@ export class ExitForStatement extends Statement { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.tokenToSourceNodeWithTrivia(this.tokens.exitFor); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -322,6 +363,10 @@ export class ExitWhileStatement extends Statement { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.tokenToSourceNodeWithTrivia(this.tokens.exitWhile); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -370,6 +415,14 @@ export class FunctionStatement extends Statement implements TypedefProvider { return this.func.transpile(state, nameToken); } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.arrayToSourceNodeWithTrivia(this.annotations), + //the FunctionExpression looks upwards for its name, so we don't need to include it here + this.func.toSourceNode(state) + ); + } + getTypedef(state: BrsTranspileState) { let result = []; for (let annotation of this.annotations ?? []) { @@ -490,6 +543,25 @@ export class IfStatement extends Statement { return results; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + //if + state.tokenToSourceNodeWithTrivia(this.tokens.if), + //conditions + this.condition.toSourceNode(state), + //then + state.tokenToSourceNodeWithTrivia(this.tokens.then), + //then branch + this.thenBranch?.toSourceNode(state), + //else + state.tokenToSourceNodeWithTrivia(this.tokens.else), + //else branch + this.elseBranch?.toSourceNode(state), + //end if + state.tokenToSourceNodeWithTrivia(this.tokens.endIf) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'condition', visitor, options); @@ -521,6 +593,13 @@ export class IncrementStatement extends Statement { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + this.value.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.operator) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'value', visitor, options); @@ -587,6 +666,24 @@ export class PrintStatement extends Statement { return result; } + public toSourceNode(state: TranspileState): SourceNode { + let result = [ + state.tokenToSourceNodeWithTrivia(this.tokens.print) + ]; + for (const expression of this.expressions) { + if (isExpression(expression)) { + result.push( + expression.toSourceNode(state) + ); + } else { + result.push( + state.tokenToSourceNodeWithTrivia(expression) + ); + } + } + return state.toSourceNode(...result); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { //sometimes we have semicolon Tokens in the expressions list (should probably fix that...), so only walk the actual expressions @@ -630,6 +727,27 @@ export class DimStatement extends Statement { return result; } + public toSourceNode(state: TranspileState): SourceNode { + const chunks: Array = [ + state.tokenToSourceNodeWithTrivia(this.dimToken), + state.tokenToSourceNodeWithTrivia(this.identifier), + state.tokenToSourceNodeWithTrivia(this.openingSquare) + ]; + for (let i = 0; i < this.dimensions.length; i++) { + if (i > 0) { + chunks.push(', '); + } + chunks.push( + this.dimensions[i].toSourceNode(state) + ); + } + chunks.push( + state.tokenToSourceNodeWithTrivia(this.closingSquare) + ); + return state.toSourceNode(...chunks); + } + + public walk(visitor: WalkVisitor, options: WalkOptions) { if (this.dimensions?.length > 0 && options.walkMode & InternalWalkMode.walkExpressions) { walkArray(this.dimensions, visitor, options, this); @@ -659,6 +777,13 @@ export class GotoStatement extends Statement { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.goto), + state.tokenToSourceNodeWithTrivia(this.tokens.label) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -685,6 +810,13 @@ export class LabelStatement extends Statement { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.identifier), + state.tokenToSourceNodeWithTrivia(this.tokens.colon) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -718,6 +850,13 @@ export class ReturnStatement extends Statement { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.return), + this.value?.toSourceNode(state) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'value', visitor, options); @@ -743,6 +882,12 @@ export class EndStatement extends Statement { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.end) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -766,6 +911,12 @@ export class StopStatement extends Statement { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.stop) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -834,6 +985,27 @@ export class ForStatement extends Statement { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + //for + state.tokenToSourceNodeWithTrivia(this.forToken), + //i=1 + this.counterDeclaration?.toSourceNode(state), + //to + state.tokenToSourceNodeWithTrivia(this.toToken), + //finalValue + this.finalValue?.toSourceNode(state), + //step + state.tokenToSourceNodeWithTrivia(this.stepToken), + //stepValue + this.increment?.toSourceNode(state), + //body + this.body?.toSourceNode(state), + //end for + state.tokenToSourceNodeWithTrivia(this.endForToken) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkStatements) { walk(this, 'counterDeclaration', visitor, options); @@ -901,6 +1073,23 @@ export class ForEachStatement extends Statement { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + //for each + state.tokenToSourceNodeWithTrivia(this.tokens.forEach), + //item + state.tokenToSourceNodeWithTrivia(this.item), + //in + state.tokenToSourceNodeWithTrivia(this.tokens.in), + //target + this.target?.toSourceNode(state), + //body + this.body?.toSourceNode(state), + //end for + state.tokenToSourceNodeWithTrivia(this.tokens.endFor) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'target', visitor, options); @@ -955,6 +1144,19 @@ export class WhileStatement extends Statement { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + //while + state.tokenToSourceNodeWithTrivia(this.tokens.while), + //condition + this.condition?.toSourceNode(state), + //body + this.body?.toSourceNode(state), + //end while + state.tokenToSourceNodeWithTrivia(this.tokens.endWhile) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'condition', visitor, options); @@ -969,7 +1171,9 @@ export class DottedSetStatement extends Statement { constructor( readonly obj: Expression, readonly name: Identifier, - readonly value: Expression + readonly value: Expression, + readonly dot: Token, + readonly operator: Token ) { super(); this.range = util.createRangeFromPositions(this.obj.range.start, this.value.range.end); @@ -995,6 +1199,26 @@ export class DottedSetStatement extends Statement { } } + public toSourceNode(state: TranspileState): SourceNode { + //if the value is a compound assignment, don't add the obj, dot, name, or operator...the expression will handle that + if (CompoundAssignmentOperators.includes((this.value as BinaryExpression)?.operator?.kind)) { + return this.value.toSourceNode(state); + } else { + return state.toSourceNode( + // object + this.obj.toSourceNode(state), + // . + state.tokenToSourceNodeWithTrivia(this.dot), + // name + state.tokenToSourceNodeWithTrivia(this.name), + // = + state.tokenToSourceNodeWithTrivia(this.operator), + //right-hand-side of assignment + this.value?.toSourceNode(state) + ); + } + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'obj', visitor, options); @@ -1009,7 +1233,8 @@ export class IndexedSetStatement extends Statement { readonly index: Expression, readonly value: Expression, readonly openingSquare: Token, - readonly closingSquare: Token + readonly closingSquare: Token, + readonly operator: Token ) { super(); this.range = util.createRangeFromPositions(this.obj.range.start, this.value.range.end); @@ -1039,6 +1264,27 @@ export class IndexedSetStatement extends Statement { } } + public toSourceNode(state: TranspileState): SourceNode { + //if the value is a component assignment, don't add the obj, index or operator...the expression will handle that + if (CompoundAssignmentOperators.includes((this.value as BinaryExpression)?.operator?.kind)) { + return this.value.toSourceNode(state); + } else { + return state.toSourceNode( + //obj + this.obj?.toSourceNode(state), + // [ + state.tokenToSourceNodeWithTrivia(this.openingSquare), + // index + this.index?.toSourceNode(state), + // ] + state.tokenToSourceNodeWithTrivia(this.closingSquare), + // = + state.tokenToSourceNodeWithTrivia(this.operator), + // value + this.value.toSourceNode(state) + ); + } + } walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'obj', visitor, options); @@ -1083,6 +1329,13 @@ export class LibraryStatement extends Statement implements TypedefProvider { return this.transpile(state); } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.library), + state.tokenToSourceNodeWithTrivia(this.tokens.filePath) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -1140,6 +1393,15 @@ export class NamespaceStatement extends Statement implements TypedefProvider { return this.body.transpile(state); } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.keyword), + this.nameExpression?.toSourceNode(state), + this.body?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.endKeyword) + ); + } + getTypedef(state: BrsTranspileState) { let result = [ 'namespace ', @@ -1206,6 +1468,13 @@ export class ImportStatement extends Statement implements TypedefProvider { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.importToken), + state.tokenToSourceNodeWithTrivia(this.filePathToken) + ); + } + /** * Get the typedef for this statement */ @@ -1317,6 +1586,24 @@ export class InterfaceStatement extends Statement implements TypedefProvider { return []; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.arrayToSourceNodeWithTrivia(this.annotations), + // interface + state.tokenToSourceNodeWithTrivia(this.tokens.interface), + // SomeInterface + state.tokenToSourceNodeWithTrivia(this.tokens.name), + // extends + state.tokenToSourceNodeWithTrivia(this.tokens.extends), + // SomeParentInterface + this.parentInterfaceName?.toSourceNode(state), + //interface body + state.arrayToSourceNodeWithTrivia(this.body, isCommentStatement), + //end interface + state.tokenToSourceNodeWithTrivia(this.tokens.endInterface) + ); + } + getTypedef(state: BrsTranspileState) { const result = [] as TranspileResult; for (let annotation of this.annotations ?? []) { @@ -1411,6 +1698,18 @@ export class InterfaceFieldStatement extends Statement implements TypedefProvide return this.tokens.name.text; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.arrayToSourceNodeWithTrivia(this.annotations), + //name + state.tokenToSourceNodeWithTrivia(this.tokens.name), + //as + state.tokenToSourceNodeWithTrivia(this.tokens.as), + //type + state.tokenToSourceNodeWithTrivia(this.tokens.type) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -1531,6 +1830,26 @@ export class InterfaceMethodStatement extends Statement implements TypedefProvid } return result; } + + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.arrayToSourceNodeWithTrivia(this.annotations), + // function|sub + state.tokenToSourceNodeWithTrivia(this.tokens.functionType), + // name + state.tokenToSourceNodeWithTrivia(this.tokens.name), + // ( + state.tokenToSourceNodeWithTrivia(this.tokens.leftParen), + // params + state.arrayToSourceNodeWithTrivia(this.params), + // ) + state.tokenToSourceNodeWithTrivia(this.tokens.rightParen), + // as + state.tokenToSourceNodeWithTrivia(this.tokens.as), + // + state.tokenToSourceNodeWithTrivia(this.tokens.returnType) + ); + } } export class ClassStatement extends Statement implements TypedefProvider { @@ -1607,6 +1926,18 @@ export class ClassStatement extends Statement implements TypedefProvider { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.arrayToSourceNodeWithTrivia(this.annotations), + state.tokenToSourceNodeWithTrivia(this.classKeyword), + state.tokenToSourceNodeWithTrivia(this.name), + state.tokenToSourceNodeWithTrivia(this.extendsKeyword), + this.parentClassName?.toSourceNode(state), + state.arrayToSourceNodeWithTrivia(this.body, isCommentStatement), + state.tokenToSourceNodeWithTrivia(this.end) + ); + } + getTypedef(state: BrsTranspileState) { const result = [] as TranspileResult; for (let annotation of this.annotations ?? []) { @@ -1985,6 +2316,13 @@ export class MethodStatement extends FunctionStatement { return this.func.transpile(state); } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.accessModifier), + super.toSourceNode(state) + ); + } + getTypedef(state: BrsTranspileState) { const result = [] as string[]; for (let annotation of this.annotations ?? []) { @@ -2040,7 +2378,8 @@ export class MethodStatement extends FunctionStatement { text: 'super', isReserved: false, range: state.classStatement.name.range, - leadingWhitespace: '' + leadingWhitespace: '', + leadingTrivia: [] } ), { @@ -2048,15 +2387,18 @@ export class MethodStatement extends FunctionStatement { text: '(', isReserved: false, range: state.classStatement.name.range, - leadingWhitespace: '' + leadingWhitespace: '', + leadingTrivia: [] }, { kind: TokenKind.RightParen, text: ')', isReserved: false, range: state.classStatement.name.range, - leadingWhitespace: '' + leadingWhitespace: '', + leadingTrivia: [] }, + [], [] ) ); @@ -2140,6 +2482,17 @@ export class FieldStatement extends Statement implements TypedefProvider { throw new Error('transpile not implemented for ' + Object.getPrototypeOf(this).constructor.name); } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.accessModifier), + state.tokenToSourceNodeWithTrivia(this.name), + state.tokenToSourceNodeWithTrivia(this.as), + state.tokenToSourceNodeWithTrivia(this.type), + state.tokenToSourceNodeWithTrivia(this.equal), + this.initialValue?.toSourceNode(state) + ); + } + getTypedef(state: BrsTranspileState) { const result = []; if (this.name) { @@ -2217,6 +2570,15 @@ export class TryCatchStatement extends Statement { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.try), + this.tryBranch?.toSourceNode(state), + this.catchStatement?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.tokens.endTry) + ); + } + public walk(visitor: WalkVisitor, options: WalkOptions) { if (this.tryBranch && options.walkMode & InternalWalkMode.walkStatements) { walk(this, 'tryBranch', visitor, options); @@ -2252,6 +2614,14 @@ export class CatchStatement extends Statement { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.catch), + state.tokenToSourceNodeWithTrivia(this.exceptionVariable), + this.catchBranch?.toSourceNode(state) + ); + } + public walk(visitor: WalkVisitor, options: WalkOptions) { if (this.catchBranch && options.walkMode & InternalWalkMode.walkStatements) { walk(this, 'catchBranch', visitor, options); @@ -2291,6 +2661,13 @@ export class ThrowStatement extends Statement { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.throwToken), + this.expression?.toSourceNode(state) + ); + } + public walk(visitor: WalkVisitor, options: WalkOptions) { if (this.expression && options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'expression', visitor, options); @@ -2443,6 +2820,16 @@ export class EnumStatement extends Statement implements TypedefProvider { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.arrayToSourceNodeWithTrivia(this.annotations), + state.tokenToSourceNodeWithTrivia(this.tokens.enum), + state.tokenToSourceNodeWithTrivia(this.tokens.name), + state.arrayToSourceNodeWithTrivia(this.body, isCommentStatement), + state.tokenToSourceNodeWithTrivia(this.tokens.endEnum) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkStatements) { walkArray(this.body, visitor, options, this); @@ -2496,6 +2883,15 @@ export class EnumMemberStatement extends Statement implements TypedefProvider { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.arrayToSourceNodeWithTrivia(this.annotations), + state.tokenToSourceNodeWithTrivia(this.tokens.name), + state.tokenToSourceNodeWithTrivia(this.tokens.equal), + this.value?.toSourceNode(state) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (this.value && options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'value', visitor, options); @@ -2559,6 +2955,16 @@ export class ConstStatement extends Statement implements TypedefProvider { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.arrayToSourceNodeWithTrivia(this.annotations), + state.tokenToSourceNodeWithTrivia(this.tokens.const), + state.tokenToSourceNodeWithTrivia(this.tokens.name), + state.tokenToSourceNodeWithTrivia(this.tokens.equals), + this.value?.toSourceNode(state) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (this.value && options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'value', visitor, options); @@ -2587,6 +2993,14 @@ export class ContinueStatement extends Statement { state.sourceNode(this.tokens.continue, this.tokens.loopType?.text) ]; } + + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.continue), + state.tokenToSourceNodeWithTrivia(this.tokens.loopType) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } diff --git a/src/parser/TranspileState.ts b/src/parser/TranspileState.ts index 025f76d77..fdf7d4603 100644 --- a/src/parser/TranspileState.ts +++ b/src/parser/TranspileState.ts @@ -1,6 +1,7 @@ import { SourceNode } from 'source-map'; import type { Range } from 'vscode-languageserver'; import type { BsConfig } from '../BsConfig'; +import type { Token } from '../lexer/Token'; /** * Holds the state of a transpile operation as it works its way through the transpile process @@ -65,6 +66,20 @@ export class TranspileState { ); } + /** + * Shorthand for creating a container SourceNode (one that doesn't have a location) + */ + public toSourceNode(...chunks: Array) { + return new SourceNode( + //convert 0-based range line to 1-based SourceNode line + 1, + //range and SourceNode character are both 0-based, so no conversion necessary + 0, + null, + chunks.filter(x => !!x) + ); + } + /** * Create a SourceNode from a token. This is more efficient than the above `sourceNode` function * because the entire token is passed by reference, instead of the raw string being copied to the parameter, @@ -81,6 +96,28 @@ export class TranspileState { ); } + /** + * Create a SourceNode from a token. This is more efficient than the above `sourceNode` function + * because the entire token is passed by reference, instead of the raw string being copied to the parameter, + * only to then be copied again for the SourceNode constructor + */ + public tokenToSourceNodeWithTrivia(token: { range?: Range; text: string; leadingTrivia: Token[] }) { + const result: SourceNode[] = []; + for (const item of [...token?.leadingTrivia ?? [], token]) { + if (token) { + result.push(new SourceNode( + //convert 0-based range line to 1-based SourceNode line + item.range.start.line + 1, + //range and SourceNode character are both 0-based, so no conversion necessary + item.range.start.character, + this.srcPath, + item.text + )); + } + } + return new SourceNode(1, 0, null, result); + } + /** * Create a SourceNode from a token, accounting for missing range and multi-line text */ @@ -112,4 +149,23 @@ export class TranspileState { return this.tokenToSourceNode(token); } } + + /** + * Convert an array of `toSourceNode`-enabled objects to a single SourceNode, applying an optional filter as they are iterated + * @param elements the items that have a `toSourceNode()` function + * @param filter a function to call to EXCLUDE an item. It's an exclude filter so you can pass in things like `isCommentStatement` directly + */ + public arrayToSourceNodeWithTrivia SourceNode }>(elements: Array, filter?: (element: TElement) => boolean) { + const nodes: Array = []; + if (Array.isArray(elements)) { + for (const element of elements) { + if (element && !filter?.(element)) { + nodes.push( + element?.toSourceNode(this) ?? '' + ); + } + } + } + return new SourceNode(1, 0, null, nodes); + } } diff --git a/src/parser/tests/Parser.spec.ts b/src/parser/tests/Parser.spec.ts index f6ec177ef..d965d7c6c 100644 --- a/src/parser/tests/Parser.spec.ts +++ b/src/parser/tests/Parser.spec.ts @@ -14,7 +14,8 @@ export function token(kind: TokenKind, text?: string): Token { text: text, isReserved: ReservedWords.has((text || '').toLowerCase()), range: interpolatedRange, - leadingWhitespace: '' + leadingWhitespace: '', + leadingTrivia: [] }; } diff --git a/src/parser/tests/controlFlow/For.spec.ts b/src/parser/tests/controlFlow/For.spec.ts index 0130b76d9..1829bf73e 100644 --- a/src/parser/tests/controlFlow/For.spec.ts +++ b/src/parser/tests/controlFlow/For.spec.ts @@ -103,25 +103,29 @@ describe('parser for loops', () => { kind: TokenKind.For, text: 'for', isReserved: true, - range: Range.create(0, 0, 0, 3) + range: Range.create(0, 0, 0, 3), + leadingTrivia: [] }, { kind: TokenKind.Identifier, text: 'i', isReserved: false, - range: Range.create(0, 4, 0, 5) + range: Range.create(0, 4, 0, 5), + leadingTrivia: [] }, { kind: TokenKind.Equal, text: '=', isReserved: false, - range: Range.create(0, 6, 0, 7) + range: Range.create(0, 6, 0, 7), + leadingTrivia: [] }, { kind: TokenKind.IntegerLiteral, text: '0', isReserved: false, - range: Range.create(0, 8, 0, 9) + range: Range.create(0, 8, 0, 9), + leadingTrivia: [] }, { kind: TokenKind.To, @@ -130,19 +134,22 @@ describe('parser for loops', () => { range: { start: { line: 0, character: 10 }, end: { line: 0, character: 12 } - } + }, + leadingTrivia: [] }, { kind: TokenKind.IntegerLiteral, text: '10', isReserved: false, - range: Range.create(0, 13, 0, 15) + range: Range.create(0, 13, 0, 15), + leadingTrivia: [] }, { kind: TokenKind.Newline, text: '\n', isReserved: false, - range: Range.create(0, 15, 0, 16) + range: Range.create(0, 15, 0, 16), + leadingTrivia: [] }, // loop body isn't significant for location tracking, so helper functions are safe identifier('Rnd'), @@ -154,7 +161,8 @@ describe('parser for loops', () => { kind: TokenKind.EndFor, text: 'end for', isReserved: false, - range: Range.create(2, 0, 2, 8) + range: Range.create(2, 0, 2, 8), + leadingTrivia: [] }, EOF ]); diff --git a/src/parser/tests/controlFlow/ForEach.spec.ts b/src/parser/tests/controlFlow/ForEach.spec.ts index a340992fa..f143659ad 100644 --- a/src/parser/tests/controlFlow/ForEach.spec.ts +++ b/src/parser/tests/controlFlow/ForEach.spec.ts @@ -66,31 +66,36 @@ describe('parser foreach loops', () => { kind: TokenKind.ForEach, text: 'for each', isReserved: true, - range: Range.create(0, 0, 0, 8) + range: Range.create(0, 0, 0, 8), + leadingTrivia: [] }, { kind: TokenKind.Identifier, text: 'a', isReserved: false, - range: Range.create(0, 9, 0, 10) + range: Range.create(0, 9, 0, 10), + leadingTrivia: [] }, { kind: TokenKind.Identifier, text: 'in', isReserved: true, - range: Range.create(0, 11, 0, 13) + range: Range.create(0, 11, 0, 13), + leadingTrivia: [] }, { kind: TokenKind.Identifier, text: 'b', isReserved: false, - range: Range.create(0, 14, 0, 15) + range: Range.create(0, 14, 0, 15), + leadingTrivia: [] }, { kind: TokenKind.Newline, text: '\n', isReserved: false, - range: Range.create(0, 15, 0, 16) + range: Range.create(0, 15, 0, 16), + leadingTrivia: [] }, // loop body isn't significant for location tracking, so helper functions are safe identifier('Rnd'), @@ -102,7 +107,8 @@ describe('parser foreach loops', () => { kind: TokenKind.EndFor, text: 'end for', isReserved: false, - range: Range.create(2, 0, 2, 7) + range: Range.create(2, 0, 2, 7), + leadingTrivia: [] }, EOF ]); diff --git a/src/parser/tests/controlFlow/While.spec.ts b/src/parser/tests/controlFlow/While.spec.ts index c85930281..c9d00247a 100644 --- a/src/parser/tests/controlFlow/While.spec.ts +++ b/src/parser/tests/controlFlow/While.spec.ts @@ -76,19 +76,22 @@ describe('parser while statements', () => { kind: TokenKind.While, text: 'while', isReserved: true, - range: Range.create(0, 0, 0, 5) + range: Range.create(0, 0, 0, 5), + leadingTrivia: [] }, { kind: TokenKind.True, text: 'true', isReserved: true, - range: Range.create(0, 6, 0, 10) + range: Range.create(0, 6, 0, 10), + leadingTrivia: [] }, { kind: TokenKind.Newline, text: '\n', isReserved: false, - range: Range.create(0, 10, 0, 11) + range: Range.create(0, 10, 0, 11), + leadingTrivia: [] }, // loop body isn't significant for location tracking, so helper functions are safe identifier('Rnd'), @@ -100,7 +103,8 @@ describe('parser while statements', () => { kind: TokenKind.EndWhile, text: 'end while', isReserved: false, - range: Range.create(2, 0, 2, 9) + range: Range.create(2, 0, 2, 9), + leadingTrivia: [] }, EOF ]); diff --git a/src/parser/tests/expression/Call.spec.ts b/src/parser/tests/expression/Call.spec.ts index 3d238dc23..d857ed494 100644 --- a/src/parser/tests/expression/Call.spec.ts +++ b/src/parser/tests/expression/Call.spec.ts @@ -14,7 +14,7 @@ describe('parser call expressions', () => { it('parses named function calls', () => { const { statements, diagnostics } = Parser.parse([ identifier('RebootSystem'), - { kind: TokenKind.LeftParen, text: '(', range: null }, + { kind: TokenKind.LeftParen, text: '(', range: null, leadingTrivia: [] }, token(TokenKind.RightParen, ')'), EOF ]); @@ -65,7 +65,7 @@ describe('parser call expressions', () => { it('allows closing parentheses on separate line', () => { const { statements, diagnostics } = Parser.parse([ identifier('RebootSystem'), - { kind: TokenKind.LeftParen, text: '(', range: null }, + { kind: TokenKind.LeftParen, text: '(', range: null, leadingTrivia: [] }, token(TokenKind.Newline, '\\n'), token(TokenKind.Newline, '\\n'), token(TokenKind.RightParen, ')'), @@ -128,9 +128,9 @@ describe('parser call expressions', () => { it('accepts arguments', () => { const { statements, diagnostics } = Parser.parse([ identifier('add'), - { kind: TokenKind.LeftParen, text: '(', range: null }, + { kind: TokenKind.LeftParen, text: '(', range: null, leadingTrivia: [] }, token(TokenKind.IntegerLiteral, '1'), - { kind: TokenKind.Comma, text: ',', range: null }, + { kind: TokenKind.Comma, text: ',', range: null, leadingTrivia: [] }, token(TokenKind.IntegerLiteral, '2'), token(TokenKind.RightParen, ')'), EOF diff --git a/src/parser/tests/statement/PrintStatement.spec.ts b/src/parser/tests/statement/PrintStatement.spec.ts index 4e8a172cc..4bae099e2 100644 --- a/src/parser/tests/statement/PrintStatement.spec.ts +++ b/src/parser/tests/statement/PrintStatement.spec.ts @@ -80,21 +80,24 @@ describe('parser print statements', () => { text: 'print', isReserved: true, range: Range.create(0, 0, 1, 5), - leadingWhitespace: '' + leadingWhitespace: '', + leadingTrivia: [] }, { kind: TokenKind.StringLiteral, text: `"foo"`, isReserved: false, range: Range.create(0, 6, 0, 11), - leadingWhitespace: '' + leadingWhitespace: '', + leadingTrivia: [] }, { kind: TokenKind.Eof, text: '\0', isReserved: false, range: Range.create(0, 11, 0, 12), - leadingWhitespace: '' + leadingWhitespace: '', + leadingTrivia: [] } ]); diff --git a/src/parser/tests/statement/ReturnStatement.spec.ts b/src/parser/tests/statement/ReturnStatement.spec.ts index fe1563f9e..9e0f29111 100644 --- a/src/parser/tests/statement/ReturnStatement.spec.ts +++ b/src/parser/tests/statement/ReturnStatement.spec.ts @@ -51,7 +51,7 @@ describe('parser return statements', () => { token(TokenKind.Newline, '\\n'), token(TokenKind.Return, 'return'), identifier('RebootSystem'), - { kind: TokenKind.LeftParen, text: '(', range: null }, + { kind: TokenKind.LeftParen, text: '(', range: null, leadingTrivia: [] }, token(TokenKind.RightParen, ')'), token(TokenKind.Newline, '\\n'), token(TokenKind.EndFunction, 'end function'), @@ -81,13 +81,15 @@ describe('parser return statements', () => { kind: TokenKind.Return, text: 'return', isReserved: true, - range: Range.create(1, 2, 1, 8) + range: Range.create(1, 2, 1, 8), + leadingTrivia: [] }, { kind: TokenKind.IntegerLiteral, text: '5', isReserved: false, - range: Range.create(1, 9, 1, 10) + range: Range.create(1, 9, 1, 10), + leadingTrivia: [] }, token(TokenKind.Newline, '\\n'), token(TokenKind.EndFunction, 'end function'), diff --git a/src/parser/tests/statement/Set.spec.ts b/src/parser/tests/statement/Set.spec.ts index e52797d95..cdf2737d7 100644 --- a/src/parser/tests/statement/Set.spec.ts +++ b/src/parser/tests/statement/Set.spec.ts @@ -218,7 +218,7 @@ describe('parser indexed assignment', () => { range: Range.create(1, 10, 1, 11), leadingWhitespace: '' } - ]); + ] as any[]); expect(diagnostics).to.be.empty; expect(statements).to.be.lengthOf(2);