Jm/js module linking (#98)

Added host provided source / module linking
The new feature 'js_link_source' allows for runtime provided js source code.
A new build is created 'jslinksource.wasm' which can be used to quickly create debug builds, and do module linking etc.
The API is:
get_source_len -> i32 (length of source code)
get_source(ptr: i32) -> i32 (is_gzipped boolean)

.github/workflows/release.yml

@@ -6,6 +6,42 @@ on:
       - published
 
 jobs:
+  compile_jslinksource:
+    name: compile_jslinksource
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install target
+        run: rustup target add wasm32-wasi
+
+      - name: Install wasi-sdk
+        working-directory: ts/compile/builder
+        run: make download-wasi-sdk
+
+      - name: Install wizer
+        working-directory: ts/compile/builder
+        run: cargo install wizer --all-features
+
+      - name: Make jslinksource
+        working-directory: ts/compile/builder
+        run: make jslinksource
+
+      - name: Copy wasm
+        run: cp ts/compile/builder/crates/core/target_jslinksource/wasm32-wasi/release/jsbuilder_core.wasm jslinksource.wasm
+
+      - name: Upload core binary to artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: jslinksource_engine
+          path: jslinksource.wasm
+
+      - name: Upload assets to release
+        if: github.event_name == 'release'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: gh release upload ${{ github.event.release.tag_name }} jslinksource.wasm
+
   compile_core:
     name: compile_core
     runs-on: macos-latest
@@ -150,6 +186,11 @@ jobs:
         name: jsbuilder-x86_64-windows.gz
         path: releases/
 
+    - uses: actions/download-artifact@v3
+      with:
+        name: jslinksource_engine
+        path: releases/
+
     - name: Create version file
       run: echo ${{ github.event.release.tag_name }} > releases/jsbuilder-version.txt
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loopholelabs/scale",
-  "version": "0.3.17",
+  "version": "0.3.18",
   "description": "Scale is a highly-performant WebAssembly function runtime that enables composable, language-agnostic software development.",
   "source": "ts/index.ts",
   "types": "types.d.ts",

ts/compile/builder/Makefile

@@ -23,6 +23,11 @@ jssource:
 				&& cargo build --release --target=wasm32-wasi --no-default-features --features js_source --target-dir target_jssource \
 				&& cd -
 
+jslinksource:
+		cd crates/core \
+				&& cargo build --release --target=wasm32-wasi --no-default-features --features js_link_source --target-dir target_jslinksource \
+				&& cd -
+
 clean: clean-wasi-sdk clean-cargo
 
 clean-cargo:

ts/compile/builder/crates/core/Cargo.toml

@@ -18,3 +18,4 @@ flate2 = "1.0.25"
 default = ["wizer_opt"]
 wizer_opt = []
 js_source = []
+js_link_source = []

ts/compile/builder/crates/core/src/main.rs

@@ -1,3 +1,5 @@
+#![allow(unused_imports)]
+
 extern crate quickjs_wasm_sys;
 extern crate once_cell;
 extern crate polyglot_rs;
@@ -18,6 +20,9 @@ use std::os::raw::{c_int, c_void};
 #[cfg(feature = "js_source")]
 use flate2::read::GzDecoder;
 
+#[cfg(feature = "js_link_source")]
+use flate2::read::GzDecoder;
+
 use std::str;
 
 use std::io::{self, Cursor, Read, Write};
@@ -63,7 +68,7 @@ fn nextwrap(context: *mut JSContext, _jsval1: JSValue, _int1: c_int, jsval2: *mu
 fn getaddrwrap(context: *mut JSContext, _jsval1: JSValue, _int1: c_int, jsval2: *mut JSValue, _int2: c_int) -> JSValue {
   unsafe {
     let mut len = 0;    
-    let addr = JS_GetArrayBuffer(context, &mut len, *jsval2) as i32;    
+    let addr = JS_GetArrayBuffer(context, &mut len, *jsval2) as i32;
     return JS_NewInt32_Ext(context, addr);
   }
 }
@@ -75,6 +80,21 @@ pub extern "C" fn init() {
   js_init();
 }
 
+// For runtime source / linking
+#[cfg(feature = "js_link_source")]
+#[link(wasm_import_module = "env")]
+extern "C" {
+    #[link_name = "get_source_len"]
+    fn get_source_len() -> u32;
+}
+
+#[cfg(feature = "js_link_source")]
+#[link(wasm_import_module = "env")]
+extern "C" {
+    #[link_name = "get_source"]
+    fn get_source(ptr: *mut u8) -> u32;
+}
+
 fn js_init() {
   unsafe {
     let runtime = JS_NewRuntime();
@@ -86,6 +106,7 @@ fn js_init() {
       panic!("Couldn't create JavaScript context");
     }
 
+    #[allow(unused_assignments)]
     let mut js_contents = String::new();
 
     // If we are building with js_source, we should include the data. If it ends with .gz then unzip it.
@@ -102,8 +123,25 @@ fn js_init() {
       }
     }
 
+    // If we are going to do module linking, or provide the source from host
+    #[cfg(feature = "js_link_source")]
+    {
+
+      let len = get_source_len() as usize;
+      let buff = vec![0; len];
+      let is_gz = get_source(buff.as_ptr() as *mut u8);
+      let data = buff.as_slice();
+
+      if is_gz==1 {
+        let mut gz = GzDecoder::new(&data[..]);
+        gz.read_to_string(&mut js_contents).unwrap();
+      } else {
+        js_contents = str::from_utf8(data).unwrap().to_string();
+      }
+    }
+
     // If we are not building with js_source, we will read the javascript from stdin, which comes from the cli tool
-    #[cfg(not(feature = "js_source"))]
+    #[cfg(all(not(feature = "js_source"), not(feature = "js_link_source")))]
     io::stdin().read_to_string(&mut js_contents).unwrap();
 
     let len = js_contents.len() - 1;