Load WASM/JS files dynamically using importScripts (#40)

src/command_registry.ts

@@ -2,12 +2,19 @@ import { ICommandRunner } from './commands/command_runner';
 import { CoreutilsCommandRunner } from './commands/coreutils_command_runner';
 import { GrepCommandRunner } from './commands/grep_command_runner';
 import * as AllBuiltinCommands from './builtin';
+import { WasmLoader } from './wasm_loader';
 
 export class CommandRegistry {
-  private constructor() {
-    this._commandRunners = [new CoreutilsCommandRunner(), new GrepCommandRunner()];
+  constructor(wasmLoader: WasmLoader) {
+    this.registerBuiltinCommands(AllBuiltinCommands);
+
+    this._commandRunners = [
+      new CoreutilsCommandRunner(wasmLoader),
+      new GrepCommandRunner(wasmLoader)
+    ];
 
     // Command name -> runner mapping
+    // Should probably check not overwriting any command names
     for (const runner of this._commandRunners) {
       for (const name of runner.names()) {
         this._map.set(name, runner);
@@ -19,13 +26,6 @@ export class CommandRegistry {
     return this._map.get(name) ?? null;
   }
 
-  static instance(): CommandRegistry {
-    if (!CommandRegistry._instance) {
-      CommandRegistry._instance = new CommandRegistry();
-    }
-    return CommandRegistry._instance;
-  }
-
   match(start: string): string[] {
     return [...this._map.keys()].filter(name => name.startsWith(start)).sort();
   }
@@ -55,8 +55,4 @@ export class CommandRegistry {
 
   private _commandRunners: ICommandRunner[];
   private _map: Map<string, ICommandRunner> = new Map();
-
-  private static _instance: CommandRegistry;
 }
-
-CommandRegistry.instance().registerBuiltinCommands(AllBuiltinCommands);

src/commands/coreutils_command_runner.ts

@@ -1,7 +1,15 @@
-import * as CoreutilsModule from '../wasm/coreutils';
 import { WasmCommandRunner } from './wasm_command_runner';
+import { WasmLoader } from '../wasm_loader';
 
 export class CoreutilsCommandRunner extends WasmCommandRunner {
+  constructor(wasmLoader: WasmLoader) {
+    super(wasmLoader);
+  }
+
+  moduleName(): string {
+    return 'coreutils';
+  }
+
   names(): string[] {
     return [
       'basename',
@@ -50,8 +58,4 @@ export class CoreutilsCommandRunner extends WasmCommandRunner {
       'wc'
     ];
   }
-
-  protected _getWasmModule(): any {
-    return CoreutilsModule.default;
-  }
 }

src/commands/grep_command_runner.ts

@@ -1,12 +1,16 @@
-import * as GrepModule from '../wasm/grep';
 import { WasmCommandRunner } from './wasm_command_runner';
+import { WasmLoader } from '../wasm_loader';
 
 export class GrepCommandRunner extends WasmCommandRunner {
-  names(): string[] {
-    return ['grep'];
+  constructor(wasmLoader: WasmLoader) {
+    super(wasmLoader);
+  }
+
+  moduleName(): string {
+    return 'grep';
   }
 
-  protected _getWasmModule(): any {
-    return GrepModule.default;
+  names(): string[] {
+    return ['grep'];
   }
 }

src/commands/wasm_command_runner.ts

@@ -1,17 +1,20 @@
 import { ICommandRunner } from './command_runner';
 import { Context } from '../context';
 import { RunCommandError } from '../error_exit_code';
+import { WasmLoader } from '../wasm_loader';
 
 export abstract class WasmCommandRunner implements ICommandRunner {
+  constructor(readonly wasmLoader: WasmLoader) {}
+
+  abstract moduleName(): string;
+
   abstract names(): string[];
 
   async run(cmdName: string, context: Context): Promise<number> {
     const { args, fileSystem, mountpoint, stdin, stdout, stderr } = context;
 
     const start = Date.now();
-    if (!this._wasmModule) {
-      this._wasmModule = this._getWasmModule();
-    }
+    const wasmModule = this.wasmLoader.getModule(this.moduleName());
 
     // Functions for monkey-patching.
     function getChar(tty: any) {
@@ -33,7 +36,7 @@ export abstract class WasmCommandRunner implements ICommandRunner {
       ];
     }
 
-    const wasm = await this._wasmModule({
+    const wasm = await wasmModule({
       thisProgram: cmdName,
       noInitialRun: true,
       print: (text: string) => stdout.write(`${text}\n`),
@@ -89,8 +92,4 @@ export abstract class WasmCommandRunner implements ICommandRunner {
     console.log(`${cmdName} load time ${loaded - start} ms, run time ${end - loaded} ms`);
     return exitCode;
   }
-
-  protected abstract _getWasmModule(): any;
-
-  private _wasmModule: any;
 }

src/defs.ts

@@ -5,6 +5,7 @@ import { ProxyMarked, Remote } from 'comlink';
 interface IOptionsCommon {
   color?: boolean;
   mountpoint?: string;
+  wasmBaseUrl?: string;
   driveFsBaseUrl?: string;
   // Initial directories and files to create, for testing purposes.
   initialDirectories?: string[];

src/shell.ts

@@ -17,10 +17,19 @@ export class Shell implements IShell {
     this._worker = new Worker(new URL('./shell_worker.js', import.meta.url), { type: 'module' });
 
     this._remote = wrap(this._worker);
-    const { color, mountpoint, driveFsBaseUrl, initialDirectories, initialFiles } = options;
+    const { color, mountpoint, wasmBaseUrl, driveFsBaseUrl, initialDirectories, initialFiles } =
+      options;
     const { sharedArrayBuffer } = this._bufferedStdin;
     await this._remote.initialize(
-      { color, mountpoint, driveFsBaseUrl, sharedArrayBuffer, initialDirectories, initialFiles },
+      {
+        color,
+        mountpoint,
+        wasmBaseUrl,
+        driveFsBaseUrl,
+        sharedArrayBuffer,
+        initialDirectories,
+        initialFiles
+      },
       proxy(options.outputCallback),
       proxy(this.enableBufferedStdinCallback.bind(this))
     );

src/shell_impl.ts

@@ -13,14 +13,16 @@ import { History } from './history';
 import { FileInput, FileOutput, IInput, IOutput, Pipe, TerminalInput, TerminalOutput } from './io';
 import { CommandNode, PipeNode, parse } from './parse';
 import { longestStartsWith, toColumns } from './utils';
-import * as FsModule from './wasm/fs';
+import { WasmLoader } from './wasm_loader';
 
 /**
  * Shell implementation.
  */
 export class ShellImpl implements IShell {
   constructor(readonly options: IShellImpl.IOptions) {
     this._environment = new Environment(options.color ?? true);
+    this._wasmLoader = new WasmLoader(options.wasmBaseUrl);
+    this._commandRegistry = new CommandRegistry(this._wasmLoader);
   }
 
   get aliases(): Aliases {
@@ -178,8 +180,10 @@ export class ShellImpl implements IShell {
   }
 
   private async _initFilesystem(): Promise<void> {
-    this._fsModule = await FsModule.default();
-    const { FS, PATH, ERRNO_CODES, PROXYFS } = this._fsModule;
+    const fsModule = this._wasmLoader.getModule('fs');
+    const module = await fsModule({});
+    const { FS, PATH, ERRNO_CODES, PROXYFS } = module;
+
     const { mountpoint } = this;
     FS.mkdir(mountpoint, 0o777);
     this._fileSystem = { FS, PATH, ERRNO_CODES, PROXYFS };
@@ -276,7 +280,7 @@ export class ShellImpl implements IShell {
     error: IOutput
   ): Promise<number> {
     const name = commandNode.name.value;
-    const runner = CommandRegistry.instance().get(name);
+    const runner = this._commandRegistry.get(name);
     if (runner === null) {
       // Give location of command in input?
       throw new FindCommandError(name);
@@ -331,7 +335,7 @@ export class ShellImpl implements IShell {
 
     let possibles: string[] = [];
     if (isCommand) {
-      const commandMatches = CommandRegistry.instance().match(lookup);
+      const commandMatches = this._commandRegistry.match(lookup);
       const aliasMatches = this._aliases.match(lookup);
       // Combine, removing duplicates, and sort.
       possibles = [...new Set([...commandMatches, ...aliasMatches])].sort();
@@ -407,10 +411,11 @@ export class ShellImpl implements IShell {
   private _currentLine: string = '';
   private _cursorIndex: number = 0;
   private _aliases = new Aliases();
+  private _commandRegistry: CommandRegistry;
   private _environment: Environment;
   private _history = new History();
+  private _wasmLoader: WasmLoader;
 
-  private _fsModule: any;
   private _fileSystem?: IFileSystem;
   private _driveFS?: DriveFS;
 }

src/shell_worker.ts

@@ -17,10 +17,12 @@ export class ShellWorker implements IShell {
     this._outputCallback = outputCallback;
     this._enableBufferedStdinCallback = enableBufferedStdinCallback;
 
-    const { color, mountpoint, driveFsBaseUrl, initialDirectories, initialFiles } = options;
+    const { color, mountpoint, wasmBaseUrl, driveFsBaseUrl, initialDirectories, initialFiles } =
+      options;
     this._shellImpl = new ShellImpl({
       color,
       mountpoint,
+      wasmBaseUrl,
       driveFsBaseUrl,
       initialDirectories,
       initialFiles,

src/wasm_loader.ts

@@ -0,0 +1,25 @@
+/**
+ * Loader of WASM modules. Once loaded, a module is cached so that it is faster to subsequently.
+ * Must be run in a WebWorker.
+ */
+export class WasmLoader {
+  constructor(wasmBaseUrl?: string) {
+    this._wasmBaseUrl = wasmBaseUrl ?? '';
+  }
+
+  public getModule(name: string): any {
+    let module = this._cache.get(name);
+    if (module === undefined) {
+      // Maybe should use @jupyterlab/coreutils.URLExt to combine URL components.
+      const url = this._wasmBaseUrl + name + '.js';
+      console.log('Importing JS/WASM from ' + url);
+      importScripts(url);
+      module = (self as any).Module;
+      this._cache.set(name, module);
+    }
+    return module;
+  }
+
+  private _wasmBaseUrl: string;
+  private _cache = new Map<string, any>();
+}

test/package.json

@@ -8,7 +8,8 @@
     "types": "lib/index.d.ts",
     "private": true,
     "scripts": {
-        "build": "rspack build",
+        "build": "npm run build:wasm && rspack build",
+        "build:wasm": "cp node_modules/@jupyterlite/cockle/src/wasm/*.js assets",
         "serve": "rspack serve",
         "test": "playwright test",
         "test:ui": "playwright test --ui",

test/serve/index.ts

@@ -10,8 +10,7 @@ async function setup() {
     tokenize
   };
 
-  // @ts-expect-error Assigning to globalThis.
-  globalThis.cockle = cockle;
+  (globalThis as any).cockle = cockle;
 }
 
 setup();

test/serve/shell_setup.ts

@@ -40,8 +40,7 @@ async function _shell_setup_common(options: IOptions, level: number): Promise<IS
 
   // Monkey patch an inputLine function to enter a sequence of characters and append a '\r'.
   // Cannot be used for multi-character ANSI escape codes.
-  // @ts-expect-error Function not in interface.
-  shell.inputLine = async (line: string) => {
+  (shell as any).inputLine = async (line: string) => {
     for (const char of line) {
       await shell.input(char);
     }