Skip to content

Commit

Permalink
Implement text-based lockfile (#15705)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dylan-conway
dylan-conway authored Dec 11, 2024
1 parent 78445c5 commit b55ca42
Show file tree
Hide file tree
Showing 26 changed files with 3,126 additions and 643 deletions.
13 changes: 7 additions & 6 deletions src/allocators.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ const FeatureFlags = @import("./feature_flags.zig");
const Environment = @import("./env.zig");
const FixedBufferAllocator = std.heap.FixedBufferAllocator;
const bun = @import("root").bun;
const OOM = bun.OOM;

pub fn isSliceInBufferT(comptime T: type, slice: []const T, buffer: []const T) bool {
return (@intFromPtr(buffer.ptr) <= @intFromPtr(slice.ptr) and
Expand Down Expand Up @@ -328,7 +329,7 @@ pub fn BSSStringList(comptime _count: usize, comptime _item_length: usize) type
return @constCast(slice);
}

pub fn appendMutable(self: *Self, comptime AppendType: type, _value: AppendType) ![]u8 {
pub fn appendMutable(self: *Self, comptime AppendType: type, _value: AppendType) OOM![]u8 {
const appended = try @call(bun.callmod_inline, append, .{ self, AppendType, _value });
return @constCast(appended);
}
Expand All @@ -337,25 +338,25 @@ pub fn BSSStringList(comptime _count: usize, comptime _item_length: usize) type
return try self.appendMutable(EmptyType, EmptyType{ .len = len });
}

pub fn printWithType(self: *Self, comptime fmt: []const u8, comptime Args: type, args: Args) ![]const u8 {
pub fn printWithType(self: *Self, comptime fmt: []const u8, comptime Args: type, args: Args) OOM![]const u8 {
var buf = try self.appendMutable(EmptyType, EmptyType{ .len = std.fmt.count(fmt, args) + 1 });
buf[buf.len - 1] = 0;
return std.fmt.bufPrint(buf.ptr[0 .. buf.len - 1], fmt, args) catch unreachable;
}

pub fn print(self: *Self, comptime fmt: []const u8, args: anytype) ![]const u8 {
pub fn print(self: *Self, comptime fmt: []const u8, args: anytype) OOM![]const u8 {
return try printWithType(self, fmt, @TypeOf(args), args);
}

pub fn append(self: *Self, comptime AppendType: type, _value: AppendType) ![]const u8 {
pub fn append(self: *Self, comptime AppendType: type, _value: AppendType) OOM![]const u8 {
self.mutex.lock();
defer self.mutex.unlock();

return try self.doAppend(AppendType, _value);
}

threadlocal var lowercase_append_buf: bun.PathBuffer = undefined;
pub fn appendLowerCase(self: *Self, comptime AppendType: type, _value: AppendType) ![]const u8 {
pub fn appendLowerCase(self: *Self, comptime AppendType: type, _value: AppendType) OOM![]const u8 {
self.mutex.lock();
defer self.mutex.unlock();

Expand All @@ -374,7 +375,7 @@ pub fn BSSStringList(comptime _count: usize, comptime _item_length: usize) type
self: *Self,
comptime AppendType: type,
_value: AppendType,
) ![]const u8 {
) OOM![]const u8 {
const value_len: usize = brk: {
switch (comptime AppendType) {
EmptyType, []const u8, []u8, [:0]const u8, [:0]u8 => {
Expand Down
6 changes: 3 additions & 3 deletions src/bun.js/module_loader.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,9 +184,9 @@ fn dumpSourceStringFailiable(vm: *VirtualMachine, specifier: string, written: []
\\ "mappings": "{}"
\\}}
, .{
bun.fmt.formatJSONStringUTF8(std.fs.path.basename(specifier)),
bun.fmt.formatJSONStringUTF8(specifier),
bun.fmt.formatJSONStringUTF8(source_file),
bun.fmt.formatJSONStringUTF8(std.fs.path.basename(specifier), .{}),
bun.fmt.formatJSONStringUTF8(specifier, .{}),
bun.fmt.formatJSONStringUTF8(source_file, .{}),
mappings.formatVLQs(),
});
try bufw.flush();
Expand Down
11 changes: 10 additions & 1 deletion src/cli.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ const js_ast = bun.JSAst;
const linker = @import("linker.zig");
const RegularExpression = bun.RegularExpression;
const builtin = @import("builtin");
const File = bun.sys.File;

const debug = Output.scoped(.CLI, true);

Expand Down Expand Up @@ -2133,7 +2134,15 @@ pub const Command = struct {
if (strings.eqlComptime(extension, ".lockb")) {
for (bun.argv) |arg| {
if (strings.eqlComptime(arg, "--hash")) {
try PackageManagerCommand.printHash(ctx, ctx.args.entry_points[0]);
var path_buf: bun.PathBuffer = undefined;
@memcpy(path_buf[0..ctx.args.entry_points[0].len], ctx.args.entry_points[0]);
path_buf[ctx.args.entry_points[0].len] = 0;
const lockfile_path = path_buf[0..ctx.args.entry_points[0].len :0];
const file = File.open(lockfile_path, bun.O.RDONLY, 0).unwrap() catch |err| {
Output.err(err, "failed to open lockfile", .{});
Global.crash();
};
try PackageManagerCommand.printHash(ctx, file);
return;
}
}
Expand Down
3 changes: 1 addition & 2 deletions src/cli/outdated_command.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,10 @@ pub const OutdatedCommand = struct {
}

fn outdated(ctx: Command.Context, original_cwd: string, manager: *PackageManager, comptime log_level: PackageManager.Options.LogLevel) !void {
const load_lockfile_result = manager.lockfile.loadFromDisk(
const load_lockfile_result = manager.lockfile.loadFromCwd(
manager,
manager.allocator,
manager.log,
manager.options.lockfile_path,
true,
);

Expand Down
3 changes: 1 addition & 2 deletions src/cli/pack_command.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,11 +102,10 @@ pub const PackCommand = struct {
Output.flush();

var lockfile: Lockfile = undefined;
const load_from_disk_result = lockfile.loadFromDisk(
const load_from_disk_result = lockfile.loadFromCwd(
manager,
manager.allocator,
manager.log,
manager.options.lockfile_path,
false,
);

Expand Down
62 changes: 38 additions & 24 deletions src/cli/package_manager_command.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ const PackageID = Install.PackageID;
const DependencyID = Install.DependencyID;
const PackageManager = Install.PackageManager;
const Lockfile = @import("../install/lockfile.zig");
const NodeModulesFolder = Lockfile.Tree.NodeModulesFolder;
const NodeModulesFolder = Lockfile.Tree.Iterator(.node_modules).Next;
const Path = @import("../resolver/resolve_path.zig");
const String = @import("../install/semver.zig").String;
const ArrayIdentityContext = bun.ArrayIdentityContext;
Expand All @@ -26,6 +26,7 @@ const DefaultTrustedCommand = @import("./pm_trusted_command.zig").DefaultTrusted
const Environment = bun.Environment;
pub const PackCommand = @import("./pack_command.zig").PackCommand;
const Npm = Install.Npm;
const File = bun.sys.File;

const ByName = struct {
dependencies: []const Dependency,
Expand All @@ -41,7 +42,7 @@ const ByName = struct {
};

pub const PackageManagerCommand = struct {
pub fn handleLoadLockfileErrors(load_lockfile: Lockfile.LoadFromDiskResult, pm: *PackageManager) void {
pub fn handleLoadLockfileErrors(load_lockfile: Lockfile.LoadResult, pm: *PackageManager) void {
if (load_lockfile == .not_found) {
if (pm.options.log_level != .silent) {
Output.errGeneric("Lockfile not found", .{});
Expand All @@ -57,17 +58,20 @@ pub const PackageManagerCommand = struct {
}
}

pub fn printHash(ctx: Command.Context, lockfile_: []const u8) !void {
pub fn printHash(ctx: Command.Context, file: File) !void {
@setCold(true);
var lockfile_buffer: bun.PathBuffer = undefined;
@memcpy(lockfile_buffer[0..lockfile_.len], lockfile_);
lockfile_buffer[lockfile_.len] = 0;
const lockfile = lockfile_buffer[0..lockfile_.len :0];

const cli = try PackageManager.CommandLineArguments.parse(ctx.allocator, .pm);
var pm, const cwd = try PackageManager.init(ctx, cli, PackageManager.Subcommand.pm);
defer ctx.allocator.free(cwd);

const load_lockfile = pm.lockfile.loadFromDisk(pm, ctx.allocator, ctx.log, lockfile, true);
const bytes = file.readToEnd(ctx.allocator).unwrap() catch |err| {
Output.err(err, "failed to read lockfile", .{});
Global.crash();
};

const load_lockfile = pm.lockfile.loadFromBytes(pm, bytes, ctx.allocator, ctx.log);

handleLoadLockfileErrors(load_lockfile, pm);

Output.flush();
Expand Down Expand Up @@ -198,7 +202,7 @@ pub const PackageManagerCommand = struct {
Output.flush();
return;
} else if (strings.eqlComptime(subcommand, "hash")) {
const load_lockfile = pm.lockfile.loadFromDisk(pm, ctx.allocator, ctx.log, "bun.lockb", true);
const load_lockfile = pm.lockfile.loadFromCwd(pm, ctx.allocator, ctx.log, true);
handleLoadLockfileErrors(load_lockfile, pm);

_ = try pm.lockfile.hasMetaHashChanged(false, pm.lockfile.packages.len);
Expand All @@ -209,7 +213,7 @@ pub const PackageManagerCommand = struct {
Output.enableBuffering();
Global.exit(0);
} else if (strings.eqlComptime(subcommand, "hash-print")) {
const load_lockfile = pm.lockfile.loadFromDisk(pm, ctx.allocator, ctx.log, "bun.lockb", true);
const load_lockfile = pm.lockfile.loadFromCwd(pm, ctx.allocator, ctx.log, true);
handleLoadLockfileErrors(load_lockfile, pm);

Output.flush();
Expand All @@ -218,7 +222,7 @@ pub const PackageManagerCommand = struct {
Output.enableBuffering();
Global.exit(0);
} else if (strings.eqlComptime(subcommand, "hash-string")) {
const load_lockfile = pm.lockfile.loadFromDisk(pm, ctx.allocator, ctx.log, "bun.lockb", true);
const load_lockfile = pm.lockfile.loadFromCwd(pm, ctx.allocator, ctx.log, true);
handleLoadLockfileErrors(load_lockfile, pm);

_ = try pm.lockfile.hasMetaHashChanged(true, pm.lockfile.packages.len);
Expand Down Expand Up @@ -291,19 +295,19 @@ pub const PackageManagerCommand = struct {
try TrustCommand.exec(ctx, pm, args);
Global.exit(0);
} else if (strings.eqlComptime(subcommand, "ls")) {
const load_lockfile = pm.lockfile.loadFromDisk(pm, ctx.allocator, ctx.log, "bun.lockb", true);
const load_lockfile = pm.lockfile.loadFromCwd(pm, ctx.allocator, ctx.log, true);
handleLoadLockfileErrors(load_lockfile, pm);

Output.flush();
Output.disableBuffering();
const lockfile = load_lockfile.ok.lockfile;
var iterator = Lockfile.Tree.Iterator.init(lockfile);
var iterator = Lockfile.Tree.Iterator(.node_modules).init(lockfile);

var max_depth: usize = 0;

var directories = std.ArrayList(NodeModulesFolder).init(ctx.allocator);
defer directories.deinit();
while (iterator.nextNodeModulesFolder(null)) |node_modules| {
while (iterator.next(null)) |node_modules| {
const path_len = node_modules.relative_path.len;
const path = try ctx.allocator.alloc(u8, path_len + 1);
bun.copy(u8, path, node_modules.relative_path);
Expand Down Expand Up @@ -341,7 +345,7 @@ pub const PackageManagerCommand = struct {
const resolutions = slice.items(.resolution);
const root_deps = slice.items(.dependencies)[0];

Output.println("{s} node_modules ({d})", .{ path, dependencies.len });
Output.println("{s} node_modules ({d})", .{ path, lockfile.buffers.hoisted_dependencies.items.len });
const string_bytes = lockfile.buffers.string_bytes.items;
const sorted_dependencies = try ctx.allocator.alloc(DependencyID, root_deps.len);
defer ctx.allocator.free(sorted_dependencies);
Expand Down Expand Up @@ -369,21 +373,29 @@ pub const PackageManagerCommand = struct {

Global.exit(0);
} else if (strings.eqlComptime(subcommand, "migrate")) {
if (!pm.options.enable.force_save_lockfile) try_load_bun: {
std.fs.cwd().accessZ("bun.lockb", .{ .mode = .read_only }) catch break :try_load_bun;
if (!pm.options.enable.force_save_lockfile) {
if (bun.sys.existsZ("bun.lock")) {
Output.prettyErrorln(
\\<r><red>error<r>: bun.lock already exists
\\run with --force to overwrite
, .{});
Global.exit(1);
}

Output.prettyErrorln(
\\<r><red>error<r>: bun.lockb already exists
\\run with --force to overwrite
, .{});
Global.exit(1);
if (bun.sys.existsZ("bun.lockb")) {
Output.prettyErrorln(
\\<r><red>error<r>: bun.lockb already exists
\\run with --force to overwrite
, .{});
Global.exit(1);
}
}
const load_lockfile = @import("../install/migration.zig").detectAndLoadOtherLockfile(
pm.lockfile,
bun.FD.cwd(),
pm,
ctx.allocator,
pm.log,
pm.options.lockfile_path,
);
if (load_lockfile == .not_found) {
Output.prettyErrorln(
Expand All @@ -393,7 +405,9 @@ pub const PackageManagerCommand = struct {
}
handleLoadLockfileErrors(load_lockfile, pm);
const lockfile = load_lockfile.ok.lockfile;
lockfile.saveToDisk(pm.options.lockfile_path, pm.options.log_level.isVerbose());

const save_format: Lockfile.LoadResult.LockfileFormat = if (pm.options.save_text_lockfile) .text else .binary;
lockfile.saveToDisk(save_format, pm.options.log_level.isVerbose());
Global.exit(0);
}

Expand Down
35 changes: 18 additions & 17 deletions src/cli/pm_trusted_command.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,11 @@ pub const UntrustedCommand = struct {
Output.prettyError("<r><b>bun pm untrusted <r><d>v" ++ Global.package_json_version_with_sha ++ "<r>\n\n", .{});
Output.flush();

const load_lockfile = pm.lockfile.loadFromDisk(pm, ctx.allocator, ctx.log, "bun.lockb", true);
const load_lockfile = pm.lockfile.loadFromCwd(pm, ctx.allocator, ctx.log, true);
PackageManagerCommand.handleLoadLockfileErrors(load_lockfile, pm);
try pm.updateLockfileIfNeeded(load_lockfile);

const packages = pm.lockfile.packages.slice();
const metas: []Lockfile.Package.Meta = packages.items(.meta);
const scripts: []Lockfile.Package.Scripts = packages.items(.scripts);
const resolutions: []Install.Resolution = packages.items(.resolution);
const buf = pm.lockfile.buffers.string_bytes.items;
Expand All @@ -59,10 +58,8 @@ pub const UntrustedCommand = struct {
// called alias because a dependency name is not always the package name
const alias = dep.name.slice(buf);

if (metas[package_id].hasInstallScript()) {
if (!pm.lockfile.hasTrustedDependency(alias)) {
try untrusted_dep_ids.put(ctx.allocator, dep_id, {});
}
if (!pm.lockfile.hasTrustedDependency(alias)) {
try untrusted_dep_ids.put(ctx.allocator, dep_id, {});
}
}

Expand All @@ -74,15 +71,15 @@ pub const UntrustedCommand = struct {
var untrusted_deps: std.AutoArrayHashMapUnmanaged(DependencyID, Lockfile.Package.Scripts.List) = .{};
defer untrusted_deps.deinit(ctx.allocator);

var tree_iterator = Lockfile.Tree.Iterator.init(pm.lockfile);
var tree_iterator = Lockfile.Tree.Iterator(.node_modules).init(pm.lockfile);

const top_level_without_trailing_slash = strings.withoutTrailingSlash(Fs.FileSystem.instance.top_level_dir);
var abs_node_modules_path: std.ArrayListUnmanaged(u8) = .{};
defer abs_node_modules_path.deinit(ctx.allocator);
try abs_node_modules_path.appendSlice(ctx.allocator, top_level_without_trailing_slash);
try abs_node_modules_path.append(ctx.allocator, std.fs.path.sep);

while (tree_iterator.nextNodeModulesFolder(null)) |node_modules| {
while (tree_iterator.next(null)) |node_modules| {
// + 1 because we want to keep the path separator
abs_node_modules_path.items.len = top_level_without_trailing_slash.len + 1;
try abs_node_modules_path.appendSlice(ctx.allocator, node_modules.relative_path);
Expand Down Expand Up @@ -187,7 +184,7 @@ pub const TrustCommand = struct {

if (args.len == 2) errorExpectedArgs();

const load_lockfile = pm.lockfile.loadFromDisk(pm, ctx.allocator, ctx.log, "bun.lockb", true);
const load_lockfile = pm.lockfile.loadFromCwd(pm, ctx.allocator, ctx.log, true);
PackageManagerCommand.handleLoadLockfileErrors(load_lockfile, pm);
try pm.updateLockfileIfNeeded(load_lockfile);

Expand All @@ -203,7 +200,6 @@ pub const TrustCommand = struct {

const buf = pm.lockfile.buffers.string_bytes.items;
const packages = pm.lockfile.packages.slice();
const metas: []Lockfile.Package.Meta = packages.items(.meta);
const resolutions: []Install.Resolution = packages.items(.resolution);
const scripts: []Lockfile.Package.Scripts = packages.items(.scripts);

Expand All @@ -216,10 +212,8 @@ pub const TrustCommand = struct {

const alias = dep.name.slice(buf);

if (metas[package_id].hasInstallScript()) {
if (!pm.lockfile.hasTrustedDependency(alias)) {
try untrusted_dep_ids.put(ctx.allocator, dep_id, {});
}
if (!pm.lockfile.hasTrustedDependency(alias)) {
try untrusted_dep_ids.put(ctx.allocator, dep_id, {});
}
}

Expand All @@ -231,7 +225,7 @@ pub const TrustCommand = struct {
// Instead of running them right away, we group scripts by depth in the node_modules
// file structure, then run them starting at max depth. This ensures lifecycle scripts are run
// in the correct order as they would during a normal install
var tree_iter = Lockfile.Tree.Iterator.init(pm.lockfile);
var tree_iter = Lockfile.Tree.Iterator(.node_modules).init(pm.lockfile);

const top_level_without_trailing_slash = strings.withoutTrailingSlash(Fs.FileSystem.instance.top_level_dir);
var abs_node_modules_path: std.ArrayListUnmanaged(u8) = .{};
Expand All @@ -248,7 +242,7 @@ pub const TrustCommand = struct {

var scripts_count: usize = 0;

while (tree_iter.nextNodeModulesFolder(null)) |node_modules| {
while (tree_iter.next(null)) |node_modules| {
abs_node_modules_path.items.len = top_level_without_trailing_slash.len + 1;
try abs_node_modules_path.appendSlice(ctx.allocator, node_modules.relative_path);

Expand Down Expand Up @@ -423,7 +417,14 @@ pub const TrustCommand = struct {
try pm.lockfile.trusted_dependencies.?.put(ctx.allocator, @truncate(String.Builder.stringHash(name)), {});
}

pm.lockfile.saveToDisk(pm.options.lockfile_path, pm.options.log_level.isVerbose());
const save_format: Lockfile.LoadResult.LockfileFormat = if (pm.options.save_text_lockfile)
.text
else switch (load_lockfile) {
.not_found => .binary,
.err => |err| err.format,
.ok => |ok| ok.format,
};
pm.lockfile.saveToDisk(save_format, pm.options.log_level.isVerbose());

var buffer_writer = try bun.js_printer.BufferWriter.init(ctx.allocator);
try buffer_writer.buffer.list.ensureTotalCapacity(ctx.allocator, package_json_contents.len + 1);
Expand Down
Loading

0 comments on commit b55ca42

Please sign in to comment.