guybedford · guybedford · Mar 8, 2022
diff --git a/README.md b/README.md
@@ -21,6 +21,7 @@ Because we are still using the native module loader the edge cases work out comp
 * Live bindings in ES modules
 * Dynamic import expressions (`import('src/' + varname')`)
 * Circular references, with the execption that live bindings are disabled for the first unexecuted circular parent.
+* [Hot reloading extension](#hot-reloading) supporting the `import.meta.hot` API.
 
 > [Built with](https://github.com/guybedford/es-module-shims/blob/main/chompfile.toml) [Chomp](https://chompbuild.com/)
 
@@ -134,31 +135,9 @@ If a static failure is not possible and dynamic import must be used, rather use
 
 When running in polyfill mode, it can be thought of that are effectively two loaders running on the page - the ES Module Shims polyfill loader, and the native loader.
 
-Note that instances are not shared between these loaders for consistency and performance.
+Note that instances are not shared between these loaders for consistency and performance. For this reason it is important to always ensure all modules hit the polyfill path, either by having all graphs use import maps at the top-level, or via `importShim` directly.
 
-As a result, if you have two module graphs - one native and one polyfilled, they will not share the same dependency instance, for example:
-
-```html
-<script type="importmap">
-{
-  "imports": {
-    "dep": "/dep.js"
-  }
-}
-</script>
-<script type="module">
-import '/dep.js';
-</script>
-<script type="module">
-import 'dep';
-</script>
-```
-
-In the above, on browsers without import maps support, the `/dep.js` instance will be loaded natively by the first module, then the second import will fail.
-
-ES Module Shims will pick up on the second import and reexecute `/dep.js`. As a result, `/dep.js` will be executed twice on the page.
-
-For this reason it is important to always ensure all modules hit the polyfill path, either by having all graphs use import maps at the top-level, or via `importShim` directly.
+If instance sharing really is needed, the [`subgraphPassthrough: true` option](#subgraph-passthrough) can be used, although this is not recommended in production since it results in slower network performance.
 
 #### Skip Polyfill
 
@@ -434,10 +413,31 @@ var resolvedUrl = import.meta.resolve('dep', 'https://site.com/another/scope');
 
 Node.js also implements a similar API, although it's in the process of shifting to a synchronous resolver.
 
+### Hot Reloading
+
+Hot reloading support is provided via the separate `dist/hot.js` or `es-module-shims/hot` export.
+
+Load the hot reloading extension before ES Module Shims:
+
+test.html
+```html
+<script src="https://ga.jspm.io/npm:[email protected]/dist/hot.js"></script>
+<script src="https://ga.jspm.io/npm:[email protected]/dist/es-module-shims.js"></script>
+<script type="module-shim" src="/app.js"></script>
+```
+
+While the hot reloading system will work with polyfill mode, it is advisable to use shim mode for hot reloading since the `import.meta.hot` API can only be created for non-native module loads.
+
+The hot reloader will listen to websocket events at `ws://[base]/watch`. Events are strings corresponding to changed file URLs relative to the base URL. The base URL is taken from `document.baseURI`.
+
+`chomp --watch` provides a local server and websocket connection that provides this hot reloading workflow out of the box given the above `test.html` ([Chomp](https://chompbuild.com) can be installed via `npm install -g chomp`).
+
 ### Module Workers
+
 ES Module Shims can be used in module workers in browsers that provide dynamic import in worker environments, which at the moment are Chrome(80+), Edge(80+), Safari(15+).
 
 An example of ES Module Shims usage in web workers is provided below:
+
 ```js
 /**
  * 
@@ -460,6 +460,7 @@ function getWorkerScriptURL(aURL) {
 
 const worker = new Worker(getWorkerScriptURL('myEsModule.js'));
 ```
+
 > For now, in web workers must be used the non-CSP build of ES Module Shims, namely the `es-module-shim.wasm.js` output: es-module-shims/dist/es-module-shims.wasm.js.
 
 ## Init Options
@@ -478,6 +479,7 @@ Provide a `esmsInitOptions` on the global scope before `es-module-shims` is load
 * [fetch](#fetch-hook)
 * [revokeBlobURLs](#revoke-blob-urls)
 * [mapOverrides](#overriding-import-map-entries)
+* [subgraphPassthrough](#subgraph-passthrough)
 
 ```html
 <script>
@@ -498,6 +500,10 @@ window.esmsInitOptions = {
   enforceIntegrity: true, // default false
   // Permit overrides to import maps
   mapOverrides: true, // default false
+  // Ensure all natively supported subgraphs are loaded through the native loader.
+  // This is disabled by default for performance since the network fetch cache might not be shared
+  // with the native loader. Enabling it avoids some classes of instancing bugs.
+  subgraphPassthrough: true, // default false
 
   // -- Hooks --
   // Module load error
@@ -693,6 +699,48 @@ document.body.appendChild(Object.assign(document.createElement('script'), {
 
 This can be useful for HMR workflows.
 
+### Subgraph Passthrough
+
+_This option is not recommended in production for network performance reasons._
+
+It is a performance optimization in ES Module Shims that even native modules that would support execution in the native loader
+will still be executed as blob URLs through the polyfill loader in order to share the fetch cache used by ESMS itself.
+
+Otherwise, for a module graph like:
+
+main.js
+```js
+import 'dep';
+```
+
+dep.js
+```js
+console.log('dep execution');
+```
+
+`main.js` in the above uses import maps so would be polyfilled in browsers without import maps support. Then if
+the native `import '/dep.js'` were used, the native loader wouldn't have that it in its cache so we could end up
+waiting on a new network request to `/dep.js` even though the polyfill has already fetched it (since in Firefox and Safari, the module network cache and fetch cache aren't always shared). To avoid this ES Module Shims will still use a blob URL for `dep.js`
+providing the source it fetched.
+
+Setting `subgraphPassthrough: true` will cause ES Module Shims to directly inline the dependency as `import '/dep.js'`
+thus causing the native loader to fetch and execute `/dep.js` itself.
+
+This can also be useful to avoid instancing bugs, for example if `dep.js` were imported natively as well:
+
+```html
+<script type="module" src="/dep.js"></script>
+<script type="module" src="/main.js"></script>
+```
+
+In the above without `subgraphPassthrough`, `/dep.js` would be executed natively while `/main.js` would be polyfilled/
+
+When the polyfilled `/main.js` imports `/dep.js` it would be executed through the polyfill loader resulting in the
+`"dep execution"` log output being executed twice.
+
+By setting `subgraphPassthrough: true` this results in a single `"dep execution"` log - the module instance is shared
+between the native loader and the polyfill loader.
+
 ### Hooks
 
 #### Polyfill hook

diff --git a/chompfile.toml b/chompfile.toml
@@ -34,9 +34,13 @@ run = '''
   writeFileSync(process.env.TARGET, source);
 '''
 
+[[task]]
+target = 'dist/hot.js'
+run = 'cp src/hot-reload.js dist/hot.js'
+
 [[task]]
 name = 'footprint'
-deps = ['dist/es-module-shims.js', 'dist/es-module-shims.wasm.js']
+deps = ['dist/es-module-shims.js', 'dist/es-module-shims.wasm.js', 'dist/hot.js']
 template = 'footprint'
 
 [[task]]

diff --git a/package.json b/package.json
@@ -5,7 +5,8 @@
   "main": "dist/es-module-shims.js",
   "exports": {
     ".": "./dist/es-module-shims.js",
-    "./wasm": "./dist/es-module-shims.wasm.js"
+    "./wasm": "./dist/es-module-shims.wasm.js",
+    "./hot": "./dist/hot.js"
   },
   "types": "index.d.ts",
   "type": "module",

diff --git a/src/env.js b/src/env.js
@@ -8,12 +8,18 @@ const optionsScript = hasDocument ? document.querySelector('script[type=esms-opt
 export const esmsInitOptions = optionsScript ? JSON.parse(optionsScript.innerHTML) : {};
 Object.assign(esmsInitOptions, self.esmsInitOptions || {});
 
-export let shimMode = hasDocument ? !!esmsInitOptions.shimMode : true;
+export let shimMode = !!esmsInitOptions.shimMode;
 
-export const importHook = globalHook(shimMode && esmsInitOptions.onimport);
-export const resolveHook = globalHook(shimMode && esmsInitOptions.resolve);
-export let fetchHook = esmsInitOptions.fetch ? globalHook(esmsInitOptions.fetch) : fetch;
-export const metaHook = esmsInitOptions.meta ? globalHook(shimMode && esmsInitOptions.meta) : noop;
+export let importHook, resolveHook, fetchHook = fetch, metaHook;
+
+if (esmsInitOptions.onimport)
+  importHook = globalHook(esmsInitOptions.onimport);
+if (esmsInitOptions.resolve)
+  resolveHook = globalHook(esmsInitOptions.resolve);
+if (esmsInitOptions.fetch)
+  fetchHook = globalHook(esmsInitOptions.fetch);
+if (esmsInitOptions.meta)
+  metaHook = globalHook(esmsInitOptions.meta);
 
 export const skip = esmsInitOptions.skip ? new RegExp(esmsInitOptions.skip) : null;
 
@@ -31,7 +37,7 @@ export const onpolyfill = esmsInitOptions.onpolyfill ? globalHook(esmsInitOption
   console.log('%c^^ Module TypeError above is polyfilled and can be ignored ^^', 'font-weight:900;color:#391');
 };
 
-export const { revokeBlobURLs, noLoadEventRetriggers, enforceIntegrity } = esmsInitOptions;
+export const { revokeBlobURLs, noLoadEventRetriggers, enforceIntegrity, subgraphPassthrough } = esmsInitOptions;
 
 function globalHook (name) {
   return typeof name === 'string' ? self[name] : name;

diff --git a/src/es-module-shims.js b/src/es-module-shims.js
@@ -18,6 +18,7 @@ import {
   skip,
   revokeBlobURLs,
   noLoadEventRetriggers,
+  subgraphPassthrough,
   cssModulesEnabled,
   jsonModulesEnabled,
   onpolyfill,
@@ -48,13 +49,14 @@ async function _resolve (id, parentUrl) {
   };
 }
 
-const resolve = resolveHook ? async (id, parentUrl) => {
+async function resolve (id, parentUrl) {
+  if (!resolveHook) return _resolve(id, parentUrl);
   let result = resolveHook(id, parentUrl, defaultResolve);
   // will be deprecated in next major
   if (result && result.then)
     result = await result;
   return result ? { r: result, b: !resolveIfNotPlainOrUrl(id, parentUrl) && !isURL(id) } : _resolve(id, parentUrl);
-} : _resolve;
+}
 
 // importShim('mod');
 // importShim('mod', { opts });
@@ -67,7 +69,6 @@ async function importShim (id, ...args) {
     parentUrl = pageBaseUrl;
   // needed for shim check
   await initPromise;
-  if (importHook) await importHook(id, typeof args[1] !== 'string' ? args[1] : {}, parentUrl);
   if (acceptingImportMaps || shimMode || !baselinePassthrough) {
     if (hasDocument)
       processImportMaps();
@@ -76,7 +77,7 @@ async function importShim (id, ...args) {
       acceptingImportMaps = false;
   }
   await importMapPromise;
-  return topLevelLoad((await resolve(id, parentUrl)).r, { credentials: 'same-origin' });
+  return topLevelLoad(id, parentUrl, { credentials: 'same-origin' });
 }
 
 self.importShim = importShim;
@@ -168,11 +169,12 @@ let importMapPromise = initPromise;
 let firstPolyfillLoad = true;
 let acceptingImportMaps = true;
 
-async function topLevelLoad (url, fetchOpts, source, nativelyLoaded, lastStaticLoadPromise) {
+async function topLevelLoad (url, parentUrl, fetchOpts, source, nativelyLoaded, lastStaticLoadPromise) {
+  url = (await resolve(url, parentUrl)).r;
   if (!shimMode)
     acceptingImportMaps = false;
   await importMapPromise;
-  if (importHook) await importHook(url, typeof fetchOpts !== 'string' ? fetchOpts : {}, '');
+  if (importHook) await importHook(url, typeof fetchOpts !== 'string' ? fetchOpts : {}, parentUrl);
   // early analysis opt-out - no need to even fetch if we have feature support
   if (!shimMode && baselinePassthrough) {
     // for polyfill case, only dynamic import needs a return value here, and dynamic import will never pass nativelyLoaded
@@ -187,16 +189,18 @@ async function topLevelLoad (url, fetchOpts, source, nativelyLoaded, lastStaticL
   lastLoad = undefined;
   resolveDeps(load, seen);
   await lastStaticLoadPromise;
-  if (source && !shimMode && !load.n && !self.ESMS_DEBUG) {
-    const module = await dynamicImport(createBlob(source), { errUrl: source });
-    if (revokeBlobURLs) revokeObjectURLs(Object.keys(seen));
-    return module;
-  }
-  if (firstPolyfillLoad && !shimMode && load.n && nativelyLoaded) {
-    onpolyfill();
-    firstPolyfillLoad = false;
+  if (!shimMode) {
+    if (source && !load.n && !self.ESMS_DEBUG) {
+      const module = await dynamicImport(createBlob(source), { errUrl: source });
+      if (revokeBlobURLs) revokeObjectURLs(Object.keys(seen));
+      return module;
+    }
+    if (firstPolyfillLoad && load.n && nativelyLoaded) {
+      onpolyfill();
+      firstPolyfillLoad = false;
+    }
   }
-  const module = await dynamicImport(!shimMode && !load.n && nativelyLoaded ? load.u : load.b, { errUrl: load.u });
+  const module = await dynamicImport(!shimMode && !load.n && (subgraphPassthrough || nativelyLoaded) ? load.u : load.b, { errUrl: load.u });
   // if the top-level load is a shell, run its update function
   if (load.s)
     (await dynamicImport(load.s)).u$_(module);
@@ -207,25 +211,20 @@ async function topLevelLoad (url, fetchOpts, source, nativelyLoaded, lastStaticL
 }
 
 function revokeObjectURLs(registryKeys) {
-  let batch = 0;
-  const keysLength = registryKeys.length;
-  const schedule = self.requestIdleCallback ? self.requestIdleCallback : self.requestAnimationFrame;
-  schedule(cleanup);
+  let curIdx = 0;
+  const handler = self.requestIdleCallback || self.requestAnimationFrame;
+  handler(cleanup);
   function cleanup() {
-    const batchStartIndex = batch * 100;
-    if (batchStartIndex > keysLength) return
-    for (const key of registryKeys.slice(batchStartIndex, batchStartIndex + 100)) {
+    for (const key of registryKeys.slice(curIdx, curIdx += 100)) {
       const load = registry[key];
       if (load) URL.revokeObjectURL(load.b);
     }
-    batch++;
-    schedule(cleanup);
+    if (curIdx < registryKeys.length)
+      handler(cleanup);
   }
 }
 
-function urlJsString (url) {
-  return `'${url.replace(/'/g, "\\'")}'`;
-}
+const urlJsString = url => `'${url.replace(/'/g, "\\'")}'`;
 
 let lastLoad;
 function resolveDeps (load, seen) {
@@ -236,6 +235,14 @@ function resolveDeps (load, seen) {
   for (const dep of load.d)
     resolveDeps(dep, seen);
 
+  // use direct native execution when possible
+  // load.n is therefore conservative
+  if (subgraphPassthrough && !shimMode && !load.n) {
+    load.b = lastLoad = load.u;
+    load.S = undefined;
+    return;
+  }
+
   const [imports, exports] = load.a;
 
   // "execution"
@@ -293,7 +300,7 @@ function resolveDeps (load, seen) {
       // import.meta
       else if (dynamicImportIndex === -2) {
         load.m = { url: load.r, resolve: metaResolve };
-        metaHook(load.m, load.u);
+        if (metaHook) metaHook(load.m, load.u);
         pushStringTo(start);
         resolvedSource += `importShim._r[${urlJsString(load.u)}].m`;
         lastIndex = statementEnd;
@@ -553,7 +560,7 @@ function processScript (script) {
   const isDomContentLoadedScript = domContentLoadedCnt > 0;
   if (isBlockingReadyScript) readyStateCompleteCnt++;
   if (isDomContentLoadedScript) domContentLoadedCnt++;
-  const loadPromise = topLevelLoad(script.src || pageBaseUrl, getFetchOpts(script), !script.src && script.innerHTML, !shimMode, isBlockingReadyScript && lastStaticLoadPromise).catch(throwError);
+  const loadPromise = topLevelLoad(script.src || pageBaseUrl, pageBaseUrl, getFetchOpts(script), !script.src && script.innerHTML, !shimMode, isBlockingReadyScript && lastStaticLoadPromise).catch(throwError);
   if (isBlockingReadyScript)
     lastStaticLoadPromise = loadPromise.then(readyStateCompleteCheck);
   if (isDomContentLoadedScript)

diff --git a/src/es-module-shims.polyfill.js b/src/es-module-shims.polyfill.js
@@ -0,0 +1 @@
+import './loader.js';