From 59167366e3d51653607c8a7a3395a0fbd763fc21 Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Fri, 29 Nov 2024 17:08:07 -0500 Subject: [PATCH 1/5] restore pointers --- src/js_lexer.zig | 27 ++++++++++++++++++++++----- src/js_parser.zig | 20 +++++++++++--------- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/js_lexer.zig b/src/js_lexer.zig index 0a4eb1a70348c6..b6c22f113f2c95 100644 --- a/src/js_lexer.zig +++ b/src/js_lexer.zig @@ -273,6 +273,23 @@ fn NewLexer_( // } } + pub fn restore(this: *LexerType, original: *const LexerType) void { + const all_comments = this.all_comments; + const comments_to_preserve_before = this.comments_to_preserve_before; + const temp_buffer_u16 = this.temp_buffer_u16; + this.* = original.*; + + // make sure pointers are valid + this.all_comments = all_comments; + this.comments_to_preserve_before = comments_to_preserve_before; + this.temp_buffer_u16 = temp_buffer_u16; + + // TODO: maybe shrink? + this.all_comments.items.len = original.all_comments.items.len; + this.comments_to_preserve_before.items.len = original.comments_to_preserve_before.items.len; + this.temp_buffer_u16.items.len = original.temp_buffer_u16.items.len; + } + /// Look ahead at the next n codepoints without advancing the iterator. /// If fewer than n codepoints are available, then return the remainder of the string. fn peek(it: *LexerType, n: usize) string { @@ -378,7 +395,7 @@ fn NewLexer_( // 1-3 digit octal var is_bad = false; var value: i64 = c2 - '0'; - var restore = iter; + var prev = iter; _ = iterator.next(&iter) or { if (value == 0) { @@ -395,7 +412,7 @@ fn NewLexer_( switch (c3) { '0'...'7' => { value = value * 8 + c3 - '0'; - restore = iter; + prev = iter; _ = iterator.next(&iter) or return lexer.syntaxError(); const c4 = iter.c; @@ -405,14 +422,14 @@ fn NewLexer_( if (temp < 256) { value = temp; } else { - iter = restore; + iter = prev; } }, '8', '9' => { is_bad = true; }, else => { - iter = restore; + iter = prev; }, } }, @@ -420,7 +437,7 @@ fn NewLexer_( is_bad = true; }, else => { - iter = restore; + iter = prev; }, } diff --git a/src/js_parser.zig b/src/js_parser.zig index 3c27753894d024..d762bf4be6560a 100644 --- a/src/js_parser.zig +++ b/src/js_parser.zig @@ -780,7 +780,7 @@ pub const TypeScript = struct { }; pub fn isTSArrowFnJSX(p: anytype) !bool { - var oldLexer = std.mem.toBytes(p.lexer); + const old_lexer = p.lexer; try p.lexer.next(); // Look ahead to see if this should be an arrow function instead @@ -800,7 +800,7 @@ pub const TypeScript = struct { } // Restore the lexer - p.lexer = std.mem.bytesToValue(@TypeOf(p.lexer), &oldLexer); + p.lexer.restore(&old_lexer); return is_ts_arrow_fn; } @@ -870,11 +870,13 @@ pub const TypeScript = struct { } fn lookAheadNextTokenIsOpenParenOrLessThanOrDot(p: anytype) bool { - var old_lexer = std.mem.toBytes(p.lexer); + const old_lexer = p.lexer; const old_log_disabled = p.lexer.is_log_disabled; p.lexer.is_log_disabled = true; - defer p.lexer.is_log_disabled = old_log_disabled; - defer p.lexer = std.mem.bytesToValue(@TypeOf(p.lexer), &old_lexer); + defer { + p.lexer.restore(&old_lexer); + p.lexer.is_log_disabled = old_log_disabled; + } p.lexer.next() catch {}; return switch (p.lexer.token) { @@ -12816,7 +12818,7 @@ fn NewParser_( pub const Backtracking = struct { pub inline fn lexerBacktracker(p: *P, func: anytype, comptime ReturnType: type) ReturnType { p.markTypeScriptOnly(); - var old_lexer = std.mem.toBytes(p.lexer); + const old_lexer = p.lexer; const old_log_disabled = p.lexer.is_log_disabled; p.lexer.is_log_disabled = true; defer p.lexer.is_log_disabled = old_log_disabled; @@ -12841,7 +12843,7 @@ fn NewParser_( }; if (backtrack) { - p.lexer = std.mem.bytesToValue(@TypeOf(p.lexer), &old_lexer); + p.lexer.restore(&old_lexer); if (comptime FnReturnType == anyerror!bool) { return false; @@ -12861,7 +12863,7 @@ fn NewParser_( pub inline fn lexerBacktrackerWithArgs(p: *P, func: anytype, args: anytype, comptime ReturnType: type) ReturnType { p.markTypeScriptOnly(); - var old_lexer = std.mem.toBytes(p.lexer); + const old_lexer = p.lexer; const old_log_disabled = p.lexer.is_log_disabled; p.lexer.is_log_disabled = true; @@ -12882,7 +12884,7 @@ fn NewParser_( }; if (backtrack) { - p.lexer = std.mem.bytesToValue(@TypeOf(p.lexer), &old_lexer); + p.lexer.restore(&old_lexer); if (comptime FnReturnType == anyerror!bool) { return false; } From 39695ae93d401ad42d9e2aac104dffcfd895cedf Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Fri, 29 Nov 2024 17:23:21 -0500 Subject: [PATCH 2/5] comment --- src/js_lexer.zig | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/js_lexer.zig b/src/js_lexer.zig index b6c22f113f2c95..68a1b655aa0514 100644 --- a/src/js_lexer.zig +++ b/src/js_lexer.zig @@ -284,9 +284,13 @@ fn NewLexer_( this.comments_to_preserve_before = comments_to_preserve_before; this.temp_buffer_u16 = temp_buffer_u16; - // TODO: maybe shrink? this.all_comments.items.len = original.all_comments.items.len; this.comments_to_preserve_before.items.len = original.comments_to_preserve_before.items.len; + + // TODO: maybe clone the buffer instead of copy len. `all_comments` and + // `comments_to_preserve_before` will only grow so it's safe to reset + // the length, but `temp_buffer_u16` will grow and shrink which could + // lead to weird bugs this.temp_buffer_u16.items.len = original.temp_buffer_u16.items.len; } From b70a21f6de7875a2256ec25b902162c9e41ed2bd Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Mon, 2 Dec 2024 01:49:56 -0800 Subject: [PATCH 3/5] test --- test/bundler/transpiler/fixtures/9-comments.ts | 10 ++++++++++ test/bundler/transpiler/transpiler.test.js | 18 +++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 test/bundler/transpiler/fixtures/9-comments.ts diff --git a/test/bundler/transpiler/fixtures/9-comments.ts b/test/bundler/transpiler/fixtures/9-comments.ts new file mode 100644 index 00000000000000..b730d705fc8172 --- /dev/null +++ b/test/bundler/transpiler/fixtures/9-comments.ts @@ -0,0 +1,10 @@ +var a = 0; +// 1 +// 2 +// 3 +// 4 +// 5 +// 6 +// 7 +// 8 +if (a < 9 /* 9 */) console.log("success!"); diff --git a/test/bundler/transpiler/transpiler.test.js b/test/bundler/transpiler/transpiler.test.js index 7bb5bb1987c6a4..90ec5729c1a04b 100644 --- a/test/bundler/transpiler/transpiler.test.js +++ b/test/bundler/transpiler/transpiler.test.js @@ -1,5 +1,6 @@ import { describe, expect, it } from "bun:test"; -import { hideFromStackTrace } from "harness"; +import { hideFromStackTrace, bunExe, bunEnv } from "harness"; +import { join } from "path"; describe("Bun.Transpiler", () => { const transpiler = new Bun.Transpiler({ @@ -3426,3 +3427,18 @@ describe("await can only be used inside an async function message", () => { assertError(`const foo = () => await bar();`, false); }); }); + +it("does not crash with 9 comments and typescript type skipping", () => { + const cmd = [bunExe(), "build", "--minify-identifiers", join(import.meta.dir, "fixtures", "9-comments.ts")]; + console.log({ cmd }); + const { stdout, stderr, exitCode } = Bun.spawnSync({ + cmd, + stdout: "pipe", + stderr: "pipe", + env: bunEnv, + }); + + expect(stderr.toString()).toBe(""); + expect(stdout.toString()).toContain("success!"); + expect(exitCode).toBe(0); +}); From 22eef7780d66751073045facdf944ee133fe532a Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Mon, 2 Dec 2024 02:00:35 -0800 Subject: [PATCH 4/5] assertions, comment toOwnedSlice --- src/js_lexer.zig | 10 ++++------ src/js_parser.zig | 7 +++++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/js_lexer.zig b/src/js_lexer.zig index 68a1b655aa0514..bcb354401f3237 100644 --- a/src/js_lexer.zig +++ b/src/js_lexer.zig @@ -284,14 +284,12 @@ fn NewLexer_( this.comments_to_preserve_before = comments_to_preserve_before; this.temp_buffer_u16 = temp_buffer_u16; + bun.debugAssert(all_comments.items.len >= original.all_comments.items.len); + bun.debugAssert(comments_to_preserve_before.items.len >= original.comments_to_preserve_before.items.len); + bun.debugAssert(temp_buffer_u16.items.len == 0 and original.temp_buffer_u16.items.len == 0); + this.all_comments.items.len = original.all_comments.items.len; this.comments_to_preserve_before.items.len = original.comments_to_preserve_before.items.len; - - // TODO: maybe clone the buffer instead of copy len. `all_comments` and - // `comments_to_preserve_before` will only grow so it's safe to reset - // the length, but `temp_buffer_u16` will grow and shrink which could - // lead to weird bugs - this.temp_buffer_u16.items.len = original.temp_buffer_u16.items.len; } /// Look ahead at the next n codepoints without advancing the iterator. diff --git a/src/js_parser.zig b/src/js_parser.zig index b25d94071d43d4..812ff3580f7de6 100644 --- a/src/js_parser.zig +++ b/src/js_parser.zig @@ -15428,7 +15428,10 @@ fn NewParser_( p.lexer.preserve_all_comments_before = true; try p.lexer.expect(.t_open_paren); - const comments = try p.lexer.comments_to_preserve_before.toOwnedSlice(); + + // const comments = try p.lexer.comments_to_preserve_before.toOwnedSlice(); + p.lexer.comments_to_preserve_before.clearRetainingCapacity(); + p.lexer.preserve_all_comments_before = false; const value = try p.parseExpr(.comma); @@ -15466,7 +15469,7 @@ fn NewParser_( } } - _ = comments; // TODO: leading_interior comments + // _ = comments; // TODO: leading_interior comments return p.newExpr(E.Import{ .expr = value, From 37baaef21d5c83a446fd1f29edbddacc4bdf0520 Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Mon, 2 Dec 2024 02:01:37 -0800 Subject: [PATCH 5/5] update --- test/bundler/transpiler/transpiler.test.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/bundler/transpiler/transpiler.test.js b/test/bundler/transpiler/transpiler.test.js index 90ec5729c1a04b..8757b968ae5e8a 100644 --- a/test/bundler/transpiler/transpiler.test.js +++ b/test/bundler/transpiler/transpiler.test.js @@ -3430,7 +3430,6 @@ describe("await can only be used inside an async function message", () => { it("does not crash with 9 comments and typescript type skipping", () => { const cmd = [bunExe(), "build", "--minify-identifiers", join(import.meta.dir, "fixtures", "9-comments.ts")]; - console.log({ cmd }); const { stdout, stderr, exitCode } = Bun.spawnSync({ cmd, stdout: "pipe",