From e462554b26184b1b2c4e14c875b19725355c1072 Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Sat, 21 Oct 2023 17:44:41 -0400 Subject: [PATCH] src: add `node_modules.h` for module loader --- lib/internal/modules/package_json_reader.js | 12 +- node.gyp | 2 + src/base_object_types.h | 3 +- src/node_binding.cc | 1 + src/node_binding.h | 1 + src/node_external_reference.h | 1 + src/node_file.cc | 64 -------- src/node_modules.cc | 169 ++++++++++++++++++++ src/node_modules.h | 52 ++++++ src/node_snapshotable.cc | 1 + test/parallel/test-bootstrap-modules.js | 1 + typings/globals.d.ts | 2 + typings/internalBinding/fs.d.ts | 1 - typings/internalBinding/modules.d.ts | 3 + 14 files changed, 237 insertions(+), 76 deletions(-) create mode 100644 src/node_modules.cc create mode 100644 src/node_modules.h create mode 100644 typings/internalBinding/modules.d.ts diff --git a/lib/internal/modules/package_json_reader.js b/lib/internal/modules/package_json_reader.js index 65f5ce3551bbd0..dc6ff150daf2cf 100644 --- a/lib/internal/modules/package_json_reader.js +++ b/lib/internal/modules/package_json_reader.js @@ -3,7 +3,6 @@ const { JSONParse, ObjectPrototypeHasOwnProperty, - SafeMap, StringPrototypeEndsWith, StringPrototypeIndexOf, StringPrototypeLastIndexOf, @@ -12,15 +11,13 @@ const { const { ERR_INVALID_PACKAGE_CONFIG, } = require('internal/errors').codes; -const { internalModuleReadJSON } = internalBinding('fs'); +const modulesBinding = internalBinding('modules'); const { resolve, sep, toNamespacedPath } = require('path'); const permission = require('internal/process/permission'); const { kEmptyObject, setOwnProperty } = require('internal/util'); const { fileURLToPath, pathToFileURL } = require('internal/url'); -const cache = new SafeMap(); - let manifest; /** @@ -45,11 +42,7 @@ let manifest; * @returns {PackageConfig} */ function read(jsonPath, { base, specifier, isESM } = kEmptyObject) { - if (cache.has(jsonPath)) { - return cache.get(jsonPath); - } - - const string = internalModuleReadJSON( + const string = modulesBinding.readPackageJSON( toNamespacedPath(jsonPath), ); const result = { @@ -112,7 +105,6 @@ function read(jsonPath, { base, specifier, isESM } = kEmptyObject) { manifest.assertIntegrity(jsonURL, string); } } - cache.set(jsonPath, result); return result; } diff --git a/node.gyp b/node.gyp index 93e4235a0f3efd..c6afd6690d4d21 100644 --- a/node.gyp +++ b/node.gyp @@ -112,6 +112,7 @@ 'src/node_main_instance.cc', 'src/node_messaging.cc', 'src/node_metadata.cc', + 'src/node_modules.cc', 'src/node_options.cc', 'src/node_os.cc', 'src/node_perf.cc', @@ -234,6 +235,7 @@ 'src/node_messaging.h', 'src/node_metadata.h', 'src/node_mutex.h', + 'src/node_modules.h', 'src/node_object_wrap.h', 'src/node_options.h', 'src/node_options-inl.h', diff --git a/src/base_object_types.h b/src/base_object_types.h index cb034f1d62b681..9cfe6a77f71708 100644 --- a/src/base_object_types.h +++ b/src/base_object_types.h @@ -17,7 +17,8 @@ namespace node { V(blob_binding_data, BlobBindingData) \ V(process_binding_data, process::BindingData) \ V(timers_binding_data, timers::BindingData) \ - V(url_binding_data, url::BindingData) + V(url_binding_data, url::BindingData) \ + V(modules_binding_data, modules::BindingData) #define UNSERIALIZABLE_BINDING_TYPES(V) \ V(http2_binding_data, http2::BindingData) \ diff --git a/src/node_binding.cc b/src/node_binding.cc index 97257d47c61738..2b69a828a744b6 100644 --- a/src/node_binding.cc +++ b/src/node_binding.cc @@ -49,6 +49,7 @@ V(js_stream) \ V(js_udp_wrap) \ V(messaging) \ + V(modules) \ V(module_wrap) \ V(mksnapshot) \ V(options) \ diff --git a/src/node_binding.h b/src/node_binding.h index 9f0692ca4e190b..29bb478b99e8eb 100644 --- a/src/node_binding.h +++ b/src/node_binding.h @@ -38,6 +38,7 @@ static_assert(static_cast(NM_F_LINKED) == V(encoding_binding) \ V(fs) \ V(mksnapshot) \ + V(modules) \ V(timers) \ V(process_methods) \ V(performance) \ diff --git a/src/node_external_reference.h b/src/node_external_reference.h index ae37094c8e117e..a647967077967e 100644 --- a/src/node_external_reference.h +++ b/src/node_external_reference.h @@ -100,6 +100,7 @@ class ExternalReferenceRegistry { V(messaging) \ V(mksnapshot) \ V(module_wrap) \ + V(modules) \ V(options) \ V(os) \ V(performance) \ diff --git a/src/node_file.cc b/src/node_file.cc index a9b9231009c144..2579c465e4d43f 100644 --- a/src/node_file.cc +++ b/src/node_file.cc @@ -1050,68 +1050,6 @@ static void ExistsSync(const FunctionCallbackInfo& args) { args.GetReturnValue().Set(err == 0); } -// Used to speed up module loading. Returns an array [string, boolean] -static void InternalModuleReadJSON(const FunctionCallbackInfo& args) { - Environment* env = Environment::GetCurrent(args); - Isolate* isolate = env->isolate(); - uv_loop_t* loop = env->event_loop(); - - CHECK(args[0]->IsString()); - node::Utf8Value path(isolate, args[0]); - THROW_IF_INSUFFICIENT_PERMISSIONS( - env, permission::PermissionScope::kFileSystemRead, path.ToStringView()); - - if (strlen(*path) != path.length()) { - return; // Contains a nul byte. - } - uv_fs_t open_req; - const int fd = uv_fs_open(loop, &open_req, *path, O_RDONLY, 0, nullptr); - uv_fs_req_cleanup(&open_req); - - if (fd < 0) { - return; - } - - auto defer_close = OnScopeLeave([fd, loop]() { - uv_fs_t close_req; - CHECK_EQ(0, uv_fs_close(loop, &close_req, fd, nullptr)); - uv_fs_req_cleanup(&close_req); - }); - - const size_t kBlockSize = 32 << 10; - std::vector chars; - int64_t offset = 0; - ssize_t numchars; - do { - const size_t start = chars.size(); - chars.resize(start + kBlockSize); - - uv_buf_t buf; - buf.base = &chars[start]; - buf.len = kBlockSize; - - uv_fs_t read_req; - numchars = uv_fs_read(loop, &read_req, fd, &buf, 1, offset, nullptr); - uv_fs_req_cleanup(&read_req); - - if (numchars < 0) { - return; - } - offset += numchars; - } while (static_cast(numchars) == kBlockSize); - - size_t start = 0; - if (offset >= 3 && 0 == memcmp(chars.data(), "\xEF\xBB\xBF", 3)) { - start = 3; // Skip UTF-8 BOM. - } - const size_t size = offset - start; - - args.GetReturnValue().Set( - String::NewFromUtf8( - isolate, &chars[start], v8::NewStringType::kNormal, size) - .ToLocalChecked()); -} - // Used to speed up module loading. Returns 0 if the path refers to // a file, 1 when it's a directory or < 0 on error (usually -ENOENT.) // The speedup comes from not creating thousands of Stat and Error objects. @@ -3255,7 +3193,6 @@ static void CreatePerIsolateProperties(IsolateData* isolate_data, SetMethod(isolate, target, "rmdir", RMDir); SetMethod(isolate, target, "mkdir", MKDir); SetMethod(isolate, target, "readdir", ReadDir); - SetMethod(isolate, target, "internalModuleReadJSON", InternalModuleReadJSON); SetMethod(isolate, target, "internalModuleStat", InternalModuleStat); SetMethod(isolate, target, "stat", Stat); SetMethod(isolate, target, "lstat", LStat); @@ -3375,7 +3312,6 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) { registry->Register(RMDir); registry->Register(MKDir); registry->Register(ReadDir); - registry->Register(InternalModuleReadJSON); registry->Register(InternalModuleStat); registry->Register(Stat); registry->Register(LStat); diff --git a/src/node_modules.cc b/src/node_modules.cc new file mode 100644 index 00000000000000..8611c2188a453c --- /dev/null +++ b/src/node_modules.cc @@ -0,0 +1,169 @@ +#include "node_modules.h" +#include "base_object-inl.h" +#include "node_errors.h" +#include "node_external_reference.h" +#include "util-inl.h" +#include "v8-fast-api-calls.h" +#include "v8.h" + +namespace node { +namespace modules { + +using v8::CFunction; +using v8::Context; +using v8::FastOneByteString; +using v8::FunctionCallbackInfo; +using v8::HandleScope; +using v8::Isolate; +using v8::Local; +using v8::MaybeLocal; +using v8::NewStringType; +using v8::Object; +using v8::ObjectTemplate; +using v8::String; +using v8::Value; + +void BindingData::MemoryInfo(MemoryTracker* tracker) const { + tracker->TrackField("package_json_files", package_json_files_); +} + +BindingData::BindingData(Realm* realm, v8::Local object) + : SnapshotableObject(realm, object, type_int) { + package_json_files_ = v8::Map::New(realm->isolate()); + object + ->Set(realm->context(), + FIXED_ONE_BYTE_STRING(realm->isolate(), "packageJsonFiles"), + package_json_files_) + .Check(); +} + +bool BindingData::PrepareForSerialization(v8::Local context, + v8::SnapshotCreator* creator) { + // Return true because we need to maintain the reference to the binding from + // JS land. + return true; +} + +InternalFieldInfoBase* BindingData::Serialize(int index) { + DCHECK_IS_SNAPSHOT_SLOT(index); + InternalFieldInfo* info = + InternalFieldInfoBase::New(type()); + return info; +} + +void BindingData::Deserialize(v8::Local context, + v8::Local holder, + int index, + InternalFieldInfoBase* info) { + DCHECK_IS_SNAPSHOT_SLOT(index); + v8::HandleScope scope(context->GetIsolate()); + Realm* realm = Realm::GetCurrent(context); + BindingData* binding = realm->AddBindingData(holder); + CHECK_NOT_NULL(binding); +} + +void BindingData::ReadPackageJSON(const FunctionCallbackInfo& args) { + CHECK_GE(args.Length(), 1); + CHECK(args[0]->IsString()); + + Realm* realm = Realm::GetCurrent(args); + auto isolate = realm->isolate(); + auto env = realm->env(); + auto context = realm->context(); + auto binding_data = realm->GetBindingData(); + + Utf8Value path(isolate, args[0]); + THROW_IF_INSUFFICIENT_PERMISSIONS( + env, permission::PermissionScope::kFileSystemRead, path.ToStringView()); + + if (strlen(*path) != path.length()) { + return; // Contains a nul byte. + } + + Local cache_key = + ToV8Value(context, path.out(), isolate).ToLocalChecked(); + MaybeLocal maybe_existing = + binding_data->package_json_files_->Get(context, cache_key); + Local existing_value; + if (maybe_existing.ToLocal(&existing_value)) { + CHECK(existing_value->IsString()); + return args.GetReturnValue().Set(existing_value); + } + + uv_fs_t open_req; + const int fd = uv_fs_open(nullptr, &open_req, *path, O_RDONLY, 0, nullptr); + uv_fs_req_cleanup(&open_req); + + if (fd < 0) { + return; + } + + auto defer_close = OnScopeLeave([fd]() { + uv_fs_t close_req; + CHECK_EQ(0, uv_fs_close(nullptr, &close_req, fd, nullptr)); + uv_fs_req_cleanup(&close_req); + }); + + const size_t kBlockSize = 32 << 10; + std::vector chars; + int64_t offset = 0; + ssize_t numchars; + do { + const size_t start = chars.size(); + chars.resize(start + kBlockSize); + + uv_buf_t buf; + buf.base = &chars[start]; + buf.len = kBlockSize; + + uv_fs_t read_req; + numchars = uv_fs_read(nullptr, &read_req, fd, &buf, 1, offset, nullptr); + uv_fs_req_cleanup(&read_req); + + if (numchars < 0) { + return; + } + offset += numchars; + } while (static_cast(numchars) == kBlockSize); + + size_t start = 0; + if (offset >= 3 && 0 == memcmp(chars.data(), "\xEF\xBB\xBF", 3)) { + start = 3; // Skip UTF-8 BOM. + } + + auto content = + String::NewFromUtf8( + isolate, &chars[start], v8::NewStringType::kNormal, offset - start) + .ToLocalChecked(); + USE(binding_data->package_json_files_->Set(context, cache_key, content)); + args.GetReturnValue().Set(content); +} + +void BindingData::CreatePerIsolateProperties(IsolateData* isolate_data, + Local target) { + Isolate* isolate = isolate_data->isolate(); + SetMethod(isolate, target, "readPackageJSON", ReadPackageJSON); +} + +void BindingData::CreatePerContextProperties(Local target, + Local unused, + Local context, + void* priv) { + Realm* realm = Realm::GetCurrent(context); + realm->AddBindingData(target); +} + +void BindingData::RegisterExternalReferences( + ExternalReferenceRegistry* registry) { + registry->Register(ReadPackageJSON); +} + +} // namespace modules +} // namespace node + +NODE_BINDING_CONTEXT_AWARE_INTERNAL( + modules, node::modules::BindingData::CreatePerContextProperties) +NODE_BINDING_PER_ISOLATE_INIT( + modules, node::modules::BindingData::CreatePerIsolateProperties) +NODE_BINDING_EXTERNAL_REFERENCE( + modules, node::modules::BindingData::RegisterExternalReferences) diff --git a/src/node_modules.h b/src/node_modules.h new file mode 100644 index 00000000000000..c3b763eaec6e51 --- /dev/null +++ b/src/node_modules.h @@ -0,0 +1,52 @@ +#ifndef SRC_NODE_MODULES_H_ +#define SRC_NODE_MODULES_H_ + +#if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS + +#include "node.h" +#include "node_snapshotable.h" +#include "util.h" +#include "v8-fast-api-calls.h" +#include "v8.h" + +#include +#include + +namespace node { +class ExternalReferenceRegistry; + +namespace modules { + +class BindingData : public SnapshotableObject { + public: + BindingData(Realm* realm, v8::Local obj); + + using InternalFieldInfo = InternalFieldInfoBase; + + SERIALIZABLE_OBJECT_METHODS() + SET_BINDING_ID(modules_binding_data) + + void MemoryInfo(MemoryTracker* tracker) const override; + SET_SELF_SIZE(BindingData) + SET_MEMORY_INFO_NAME(BindingData) + + static void ReadPackageJSON(const v8::FunctionCallbackInfo& args); + + static void CreatePerIsolateProperties(IsolateData* isolate_data, + v8::Local ctor); + static void CreatePerContextProperties(v8::Local target, + v8::Local unused, + v8::Local context, + void* priv); + static void RegisterExternalReferences(ExternalReferenceRegistry* registry); + + private: + v8::Local package_json_files_; +}; + +} // namespace modules +} // namespace node + +#endif // defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS + +#endif // SRC_NODE_MODULES_H_ diff --git a/src/node_snapshotable.cc b/src/node_snapshotable.cc index 562a47ddcc9c8e..71d64325765048 100644 --- a/src/node_snapshotable.cc +++ b/src/node_snapshotable.cc @@ -19,6 +19,7 @@ #include "node_internals.h" #include "node_main_instance.h" #include "node_metadata.h" +#include "node_modules.h" #include "node_process.h" #include "node_snapshot_builder.h" #include "node_url.h" diff --git a/test/parallel/test-bootstrap-modules.js b/test/parallel/test-bootstrap-modules.js index 78db466a95b38e..212ef834f7d863 100644 --- a/test/parallel/test-bootstrap-modules.js +++ b/test/parallel/test-bootstrap-modules.js @@ -11,6 +11,7 @@ const assert = require('assert'); const expectedModules = new Set([ 'Internal Binding builtins', 'Internal Binding encoding_binding', + 'Internal Binding modules', 'Internal Binding errors', 'Internal Binding util', 'NativeModule internal/errors', diff --git a/typings/globals.d.ts b/typings/globals.d.ts index d72bf937bb75c9..39df64f7ec5bf4 100644 --- a/typings/globals.d.ts +++ b/typings/globals.d.ts @@ -14,6 +14,7 @@ import {TypesBinding} from "./internalBinding/types"; import {URLBinding} from "./internalBinding/url"; import {UtilBinding} from "./internalBinding/util"; import {WorkerBinding} from "./internalBinding/worker"; +import {ModulesBinding} from "./internalBinding/modules"; declare type TypedArray = | Uint8Array @@ -36,6 +37,7 @@ interface InternalBindingMap { fs: FsBinding; http_parser: HttpParserBinding; messaging: MessagingBinding; + modules: ModulesBinding; options: OptionsBinding; os: OSBinding; serdes: SerdesBinding; diff --git a/typings/internalBinding/fs.d.ts b/typings/internalBinding/fs.d.ts index 6bed01519fa2b0..611ccbec63d060 100644 --- a/typings/internalBinding/fs.d.ts +++ b/typings/internalBinding/fs.d.ts @@ -111,7 +111,6 @@ declare namespace InternalFSBinding { function futimes(fd: number, atime: number, mtime: number): void; function futimes(fd: number, atime: number, mtime: number, usePromises: typeof kUsePromises): Promise; - function internalModuleReadJSON(path: string): [] | [string, boolean]; function internalModuleStat(path: string): number; function lchown(path: string, uid: number, gid: number, req: FSReqCallback): void; diff --git a/typings/internalBinding/modules.d.ts b/typings/internalBinding/modules.d.ts new file mode 100644 index 00000000000000..b9ce3fad48bee3 --- /dev/null +++ b/typings/internalBinding/modules.d.ts @@ -0,0 +1,3 @@ +export interface ModulesBinding { + readPackageJSON(path: string): string; +}