From 4af8ebfdb19b829b620949359f70a7dcf6a6aae7 Mon Sep 17 00:00:00 2001 From: Kris Kowal Date: Fri, 2 Aug 2024 14:06:44 -0700 Subject: [PATCH 1/3] feat(import-bundle): Support endoScript --- packages/import-bundle/NEWS.md | 4 +++ packages/import-bundle/src/index.js | 31 +++++++++++++------ .../import-bundle/test/import-bundle.test.js | 25 ++++++++++++++- 3 files changed, 50 insertions(+), 10 deletions(-) diff --git a/packages/import-bundle/NEWS.md b/packages/import-bundle/NEWS.md index bb51c6f313..2d4141a5a3 100644 --- a/packages/import-bundle/NEWS.md +++ b/packages/import-bundle/NEWS.md @@ -1,5 +1,9 @@ User-visible changes to `@endo/import-bundle`: +# Next release + +- Adds support for `endoScript` format bundles. + # v1.2.1 (2024-08-01) - Fixes support for `inescapableGlobalProperties` in the `endoZipBase64` format diff --git a/packages/import-bundle/src/index.js b/packages/import-bundle/src/index.js index 81b42886c9..f8b374b35a 100644 --- a/packages/import-bundle/src/index.js +++ b/packages/import-bundle/src/index.js @@ -54,6 +54,8 @@ export async function importBundle(bundle, options = {}, powers = {}) { ); } + let compartment; + const { moduleFormat } = bundle; if (moduleFormat === 'endoZipBase64') { const { endoZipBase64 } = bundle; @@ -76,8 +78,8 @@ export async function importBundle(bundle, options = {}, powers = {}) { return namespace; } - let c; - const { source, sourceMap } = bundle; + let { source } = bundle; + const { sourceMap } = bundle; if (moduleFormat === 'getExport') { // The 'getExport' format is a string which defines a wrapper function // named `getExport()`. This function provides a `module` to the @@ -87,6 +89,7 @@ export async function importBundle(bundle, options = {}, powers = {}) { // (making it an expression). We also want to append the `sourceMap` // comment so `evaluate` can attach useful debug information. Finally, to // extract the namespace object, we need to invoke this function. + source = `(${source})\n${sourceMap || ''}`; } else if (moduleFormat === 'nestedEvaluate') { // The 'nestedEvaluate' format is similar, except the wrapper function // (now named `getExportWithNestedEvaluate`) wraps more than a single @@ -98,17 +101,27 @@ export async function importBundle(bundle, options = {}, powers = {}) { // `filePrefix`, which will be used as the sourceMap for the top-level // module. The sourceMap name for other modules will be derived from // `filePrefix` and the relative import path of each module. - endowments.nestedEvaluate = src => c.evaluate(src); + endowments.nestedEvaluate = src => compartment.evaluate(src); + source = `(${source})\n${sourceMap || ''}`; + } else if (moduleFormat === 'endoScript') { + // The 'endoScript' format is just a script. } else { Fail`unrecognized moduleFormat '${moduleFormat}'`; } - c = new CompartmentToUse(endowments, {}, { transforms }); - harden(c.globalThis); - const actualSource = `(${source})\n${sourceMap || ''}`; - const namespace = c.evaluate(actualSource)(filePrefix); - // namespace.default has the default export - return namespace; + compartment = new CompartmentToUse(endowments, {}, { transforms }); + harden(compartment.globalThis); + const result = compartment.evaluate(source); + if (moduleFormat === 'endoScript') { + // The completion value of an 'endoScript' is the namespace. + // This format does not curry the filePrefix. + return result; + } else { + // The 'getExport' and 'nestedEvaluate' formats curry a filePrefix. + const namespace = result(filePrefix); + // namespace.default has the default export + return namespace; + } } /* diff --git a/packages/import-bundle/test/import-bundle.test.js b/packages/import-bundle/test/import-bundle.test.js index 447ccdd47e..c192cac0e2 100644 --- a/packages/import-bundle/test/import-bundle.test.js +++ b/packages/import-bundle/test/import-bundle.test.js @@ -80,7 +80,7 @@ test('test import archive', async t => { const endowments = { console }; const b1EndoZip = await makeArchive( read, - new URL('bundle1.js', import.meta.url).toString(), + new URL('bundle1.js', import.meta.url).href, ); const b1EndoZipBase64 = encodeBase64(b1EndoZip); const b1EndoZipBase64Bundle = { @@ -90,6 +90,15 @@ test('test import archive', async t => { await testBundle1(t, b1EndoZipBase64Bundle, 'endoZipBase64', endowments); }); +test('test import script', async t => { + const endowments = { console }; + const b1EndoScriptBundle = await bundleSource( + url.fileURLToPath(new URL('bundle1.js', import.meta.url)), + 'endoScript', + ); + await testBundle1(t, b1EndoScriptBundle, 'endoScript', endowments); +}); + test('test missing sourceMap', async t => { function req(what) { console.log(`require(${what})`); @@ -152,6 +161,20 @@ test('inescapable global properties, nested evaluate format', async t => { t.is(ns.default, 42); }); +test('inescapable global properties, script format', async t => { + const bundle = await bundleSource( + url.fileURLToPath(new URL('export-inescapable-global.js', import.meta.url)), + 'endoScript', + ); + + const ns = await importBundle(bundle, { + inescapableGlobalProperties: { + inescapableGlobalValue: 42, + }, + }); + t.is(ns.default, 42); +}); + test('inescapable global properties, zip base64 format', async t => { const bundle = await bundleSource( url.fileURLToPath(new URL('export-inescapable-global.js', import.meta.url)), From 91a7351c0bef8d366161005c9f4f4d3bcda8e147 Mon Sep 17 00:00:00 2001 From: Kris Kowal Date: Fri, 2 Aug 2024 14:18:27 -0700 Subject: [PATCH 2/3] docs(import-bundle): Update bundle source formats, including endoScript --- packages/import-bundle/README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/import-bundle/README.md b/packages/import-bundle/README.md index 03c5cd5668..40b38e2ce4 100644 --- a/packages/import-bundle/README.md +++ b/packages/import-bundle/README.md @@ -18,13 +18,19 @@ Each call to `importBundle` creates a new `Compartment`. The globals of the new ## Module Formats -The source can be bundled in a variety of "formats". By default, `bundleSource` uses a format named `getExports`, in which the source tree is linearized into a single CommonJS-style string, and wrapped in a callable function that provides the `exports` and `module.exports` context to which the exports can be attached. `importBundle` recognizes this format, and invokes the function thus defined, to return the namespace object. +The source can be bundled in a variety of "formats". -A more sophisticated format is named `nestedEvaluate`. In this mode, the source tree is converted into a table of evaluable strings, one for each original module. This table is then encoded and wrapped as before. The evaluation process uses a separate evaluator call for each module, providing an opportunity to attach a distinct `sourceMap` to each one. This preserves relative filenames in subsequent debugging information and stack traces. +By default, `bundleSource` uses a format named `endoZipBase64`, in which the source modules and a "compartment map" are captured in a Zip file and base-64 encoded. The compartment map describes how to construct a set of [Hardened JavaScript](https://hardenedjs.org) compartments and how to load and link the source modules between them. + +The `endoScript` format captures the sources as a single JavaScript program that completes with the entry module's namespace object. + +The `getExport` format captures the sources as a single CommonJS-style string, and wrapped in a callable function that provides the `exports` and `module.exports` context to which the exports can be attached. + +More sophisticated than `getExport` is named `nestedEvaluate`. In this mode, the source tree is converted into a table of evaluable strings, one for each original module. This table is then encoded and wrapped as before. The evaluation process uses a separate evaluator call for each module, providing an opportunity to attach a distinct `sourceMap` to each one. This preserves relative filenames in subsequent debugging information and stack traces. To set a base prefix for these relative filenames, provide the `filePrefix` option. -Note that the `nestedEvaluate` format requires an endowment named `require`, although it will only be called if the source tree imported one of the few modules on the `bundle-source` "external" list. +Note that the `nestedEvaluate` format receives a global endowment named `require`, although it will only be called if the source tree imported one of the few modules on the `bundle-source` "external" list. ## Options From 029330fe607254d0d1ed0fed1dc23e15290e8185 Mon Sep 17 00:00:00 2001 From: Kris Kowal Date: Fri, 2 Aug 2024 14:20:55 -0700 Subject: [PATCH 3/3] docs(import-bundle): Wrap lines --- packages/import-bundle/README.md | 104 ++++++++++++++++++++++++------- 1 file changed, 81 insertions(+), 23 deletions(-) diff --git a/packages/import-bundle/README.md b/packages/import-bundle/README.md index 40b38e2ce4..fd07202658 100644 --- a/packages/import-bundle/README.md +++ b/packages/import-bundle/README.md @@ -1,6 +1,7 @@ # import-bundle -`importBundle` is an async function that evaluates the bundles created by `bundle-source`, turning them back into callable functions: +`importBundle` is an async function that evaluates the bundles created by +`bundle-source`, turning them back into callable functions: ```js const bundle = await bundleSource('path/to/bundle.js'); @@ -10,27 +11,53 @@ const namespace = await importBundle(bundle); const { default, namedExport1, namedExport2 } = namespace; ``` -This must be run in a SES environment: you must install SES before importing `@endo/import-bundle`. The conventional way to do this is to import a module (e.g. `@endo/init`) which does `import 'ses'; lockdown();`. +This must be run in a SES environment: you must install SES before importing +`@endo/import-bundle`. +The conventional way to do this is to import a module (e.g. `@endo/init`) which +does `import 'ses'; lockdown();`. -The bundle will be loaded into a new Compartment, which does not have access to platform globals like `document` or `Fetch` or `require`. The bundle is isolated to only having access to powerless JavaScript facilities and whatever endowments you provide. +The bundle will be loaded into a new Compartment, which does not have access to +platform globals like `document` or `Fetch` or `require`. +The bundle is isolated to only having access to powerless JavaScript facilities +and whatever endowments you provide. -Each call to `importBundle` creates a new `Compartment`. The globals of the new Compartment are frozen before any bundle code is evaluated, to enforce ocap rules. +Each call to `importBundle` creates a new `Compartment`. +The globals of the new Compartment are frozen before any bundle code is +evaluated, to enforce ocap rules. ## Module Formats The source can be bundled in a variety of "formats". -By default, `bundleSource` uses a format named `endoZipBase64`, in which the source modules and a "compartment map" are captured in a Zip file and base-64 encoded. The compartment map describes how to construct a set of [Hardened JavaScript](https://hardenedjs.org) compartments and how to load and link the source modules between them. +By default, `bundleSource` uses a format named `endoZipBase64`, in which the +source modules and a "compartment map" are captured in a Zip file and base-64 +encoded. +The compartment map describes how to construct a set of [Hardened +JavaScript](https://hardenedjs.org) compartments and how to load and link the +source modules between them. -The `endoScript` format captures the sources as a single JavaScript program that completes with the entry module's namespace object. +The `endoScript` format captures the sources as a single JavaScript program +that completes with the entry module's namespace object. -The `getExport` format captures the sources as a single CommonJS-style string, and wrapped in a callable function that provides the `exports` and `module.exports` context to which the exports can be attached. +The `getExport` format captures the sources as a single CommonJS-style string, +and wrapped in a callable function that provides the `exports` and +`module.exports` context to which the exports can be attached. -More sophisticated than `getExport` is named `nestedEvaluate`. In this mode, the source tree is converted into a table of evaluable strings, one for each original module. This table is then encoded and wrapped as before. The evaluation process uses a separate evaluator call for each module, providing an opportunity to attach a distinct `sourceMap` to each one. This preserves relative filenames in subsequent debugging information and stack traces. +More sophisticated than `getExport` is named `nestedEvaluate`. +In this mode, the source tree is converted into a table of evaluable strings, +one for each original module. +This table is then encoded and wrapped as before. +The evaluation process uses a separate evaluator call for each module, +providing an opportunity to attach a distinct `sourceMap` to each one. +This preserves relative filenames in subsequent debugging information and stack +traces. -To set a base prefix for these relative filenames, provide the `filePrefix` option. +To set a base prefix for these relative filenames, provide the `filePrefix` +option. -Note that the `nestedEvaluate` format receives a global endowment named `require`, although it will only be called if the source tree imported one of the few modules on the `bundle-source` "external" list. +Note that the `nestedEvaluate` format receives a global endowment named +`require`, although it will only be called if the source tree imported one of +the few modules on the `bundle-source` "external" list. ## Options @@ -40,19 +67,50 @@ Note that the `nestedEvaluate` format receives a global endowment named `require const namespace = await importBundle(bundle, options, powers); ``` -The most common option is `filePrefix`, which can be provided for `nestedEvaluate`-format bundles. This sets the source filename of the top-level module inside the bundle, as used in debugging messages (like the stack traces displayed in errors). The other modules will append a suffix to this filename, based upon their location within the original source tree. - -Another common option is `endowments`, which provides names that will be available everywhere in the evaluated sources. By default, the bundle will only get access to the standard JavaScript primordials (`Array`, `Object`, `Map`, etc). It will not get `document`, `window`, `Request`, `process`, `require`, or even `console` unless you provide them as endowments, giving you full control over what the loaded bundle can do. - -The `bundle-source` tool has a small number of module names marked as "external". These modules are not bundled into the source (copied from the filesystem where `bundleSource` was called). Instead, the bundler injects a call to `require()` for each external module that was imported from somewhere in original source graph. This let the final evaluation environment control what these imports get, rather than the original source tree. - -To support these "external" imports, you will need to provide a `require` endowment that can honor any such names. In addition, the `nestedEvaluate` format always needs a `require` endowment (although it will only be called if the original sources imported one of the "external" names). - -For debugging purposes, you should probably provide a `console` endowment. See `makeConsole.js` in the SwingSet source tree for inspiration. - -The rest of the `options` are passed through to the `Compartment` constructor, which currently only accepts `transforms`. For more information, see the `compartment-shim` docs in the SES repository. Note that `transforms` is defined to be an array of objects which each have a `rewrite` method. - -Note that `sloppyGlobalsMode` is only accepted by the Compartment's `evaluate` method, not the Compartment constructor itself, and thus cannot be supplied to `importBundle`. To use `sloppyGlobalsMode`, you will probably want to create a Compartment directly (and not freeze its globals). +The most common option is `filePrefix`, which can be provided for +`nestedEvaluate`-format bundles. +This sets the source filename of the top-level module inside the bundle, as +used in debugging messages (like the stack traces displayed in errors). +The other modules will append a suffix to this filename, based upon their +location within the original source tree. + +Another common option is `endowments`, which provides names that will be +available everywhere in the evaluated sources. +By default, the bundle will only get access to the standard JavaScript +primordials (`Array`, `Object`, `Map`, etc). +It will not get `document`, `window`, `Request`, `process`, `require`, or even +`console` unless you provide them as endowments, giving you full control over +what the loaded bundle can do. + +The `bundle-source` tool has a small number of module names marked as +"external". +These modules are not bundled into the source (copied from the filesystem where +`bundleSource` was called). +Instead, the bundler injects a call to `require()` for each external module +that was imported from somewhere in original source graph. +This let the final evaluation environment control what these imports get, +rather than the original source tree. + +To support these "external" imports, you will need to provide a `require` +endowment that can honor any such names. +In addition, the `nestedEvaluate` format always needs a `require` endowment +(although it will only be called if the original sources imported one of the +"external" names). + +For debugging purposes, you should probably provide a `console` endowment. +See `makeConsole.js` in the SwingSet source tree for inspiration. + +The rest of the `options` are passed through to the `Compartment` constructor, +which currently only accepts `transforms`. +For more information, see the `compartment-shim` docs in the SES repository. +Note that `transforms` is defined to be an array of objects which each have a +`rewrite` method. + +Note that `sloppyGlobalsMode` is only accepted by the Compartment's `evaluate` +method, not the Compartment constructor itself, and thus cannot be supplied to +`importBundle`. +To use `sloppyGlobalsMode`, you will probably want to create a Compartment +directly (and not freeze its globals). ## Source maps