From e53b36e08eeb07be2f56056a5db5b32a84920f7b Mon Sep 17 00:00:00 2001 From: Afshan Ahmed Khan <60838316+redoC-A2k@users.noreply.github.com> Date: Thu, 30 May 2024 18:08:09 +0530 Subject: [PATCH] Move jsonnet to wasm (#366) * Moved jsonnet ext_string , evaluateSnippet functionality from runtime to wasm * Moved jsonnet evaluateFile and vm destroy functionality from runtime to wasm * Added native function in jsonnet for quickjs-wasm * minor fixes * Implemented native async functions calling from jsonnet in wasm * minor fixes --- JS/jsonnet/src/jsonnet.js | 40 ++++ JS/jsonnet/src/lib.rs | 4 +- JS/wasm/crates/arakoo-core/Cargo.toml | 5 +- .../arakoo-core/src/apis/jsonnet/mod.rs | 218 +++++++++++++++++- JS/wasm/crates/arakoo-core/src/apis/mod.rs | 2 + JS/wasm/crates/arakoo-core/src/lib.rs | 2 +- JS/wasm/crates/serve/src/binding.rs | 38 +-- JS/wasm/crates/serve/src/lib.rs | 2 +- JS/wasm/examples/ec-wasmjs-hono/build.js | 2 +- JS/wasm/examples/ec-wasmjs-hono/src/index.js | 64 +++++ JS/wasm/wit/arakoo.wit | 2 +- JS/wasm/wit/jsonnet.wit | 10 - JS/wasm/wit/utils.wit | 3 + 13 files changed, 334 insertions(+), 58 deletions(-) delete mode 100644 JS/wasm/wit/jsonnet.wit create mode 100644 JS/wasm/wit/utils.wit diff --git a/JS/jsonnet/src/jsonnet.js b/JS/jsonnet/src/jsonnet.js index f9608c49d..802cb71f5 100644 --- a/JS/jsonnet/src/jsonnet.js +++ b/JS/jsonnet/src/jsonnet.js @@ -64,6 +64,10 @@ if (!isArakoo) { return this.vm; } + #setFunc(name, func) { + __jsonnet_func_map[name] = func; + } + evaluateSnippet(snippet) { let vm = this.#getVm(); return __jsonnet_evaluate_snippet(vm, snippet); @@ -79,6 +83,42 @@ if (!isArakoo) { let vm = this.#getVm(); return __jsonnet_evaluate_file(vm, filename); } + + javascriptCallback(name, func) { + let numOfArgs = func.length; + console.debug("Constructor name is: ", func.constructor.name); + if (func.constructor && func.constructor.name === "AsyncFunction"){ + console.debug("In if part") + if (numOfArgs > 0) { + this.#setFunc(name,async (args) => { + console.debug("Args recieved in async function: ", args) + let result = await eval(func)(...JSON.parse(args)); + return result.toString(); + }); + } else { + this.#setFunc(name, async () => { + let result = await eval(func)(); + return result; + }); + } + } else { + console.debug("In else part") + if (numOfArgs > 0) { + this.#setFunc(name, (args) => { + console.debug("Args recieved: ", args) + let result = eval(func)(...JSON.parse(args)); + return result.toString(); + }); + } else { + this.#setFunc(name, () => { + let result = eval(func)(); + return result; + }); + } + } + __jsonnet_register_func(this.vm, name, numOfArgs); + return this; + } destroy() { let vm = this.#getVm(); diff --git a/JS/jsonnet/src/lib.rs b/JS/jsonnet/src/lib.rs index 620e7076f..af68f2ffb 100755 --- a/JS/jsonnet/src/lib.rs +++ b/JS/jsonnet/src/lib.rs @@ -19,7 +19,7 @@ use std::alloc; use wasm_bindgen::prelude::*; use console_error_panic_hook; -mod context; +pub mod context; #[wasm_bindgen(module = "/read-file.js")] extern "C" { @@ -35,7 +35,7 @@ extern "C" { } pub struct VM { - state: State, + pub state: State, manifest_format: Box, trace_format: Box, tla_args: GcHashMap, diff --git a/JS/wasm/crates/arakoo-core/Cargo.toml b/JS/wasm/crates/arakoo-core/Cargo.toml index ea970497e..9e8b34c58 100644 --- a/JS/wasm/crates/arakoo-core/Cargo.toml +++ b/JS/wasm/crates/arakoo-core/Cargo.toml @@ -23,4 +23,7 @@ quickjs-wasm-rs = "3.0.0" bytes = { version = "1.6.0", features = ["serde"] } fastrand = "2.1.0" log = {version = "*"} -env_logger = {version = "*"} \ No newline at end of file +env_logger = {version = "*"} +arakoo-jsonnet ={ path = "../../../jsonnet"} +jrsonnet-gcmodule = { version = "0.3.6" } +jrsonnet-evaluator = { version = "0.5.0-pre95" } diff --git a/JS/wasm/crates/arakoo-core/src/apis/jsonnet/mod.rs b/JS/wasm/crates/arakoo-core/src/apis/jsonnet/mod.rs index 6b1474b34..9609a899b 100644 --- a/JS/wasm/crates/arakoo-core/src/apis/jsonnet/mod.rs +++ b/JS/wasm/crates/arakoo-core/src/apis/jsonnet/mod.rs @@ -1,5 +1,21 @@ +use std::{ + collections::HashMap, + ops::Deref, + sync::{Arc, Mutex}, +}; + +use crate::apis::{console, jsonnet}; + use super::{wit::edgechains, APIConfig, JSApiSet}; +use arakoo_jsonnet::{self}; use javy::quickjs::{JSContextRef, JSValue, JSValueRef}; +use jrsonnet_evaluator::{ + function::builtin::{NativeCallback, NativeCallbackHandler}, + Error, Val, +}; +use log::debug; +use quickjs_wasm_rs::{from_qjs_value, to_qjs_value}; +// use jrsonnet_evaluator::function:: pub(super) struct Jsonnet; @@ -24,6 +40,15 @@ impl JSApiSet for Jsonnet { "__jsonnet_evaluate_file", context.wrap_callback(jsonnet_evaluate_file_closure())?, )?; + let jsonnet_func_map = JSValue::Object(HashMap::new()); + global.set_property( + "__jsonnet_func_map", + to_qjs_value(context, &jsonnet_func_map)?, + )?; + global.set_property( + "__jsonnet_register_func", + context.wrap_callback(jsonnet_register_func_closure())?, + )?; global.set_property( "__jsonnet_destroy", context.wrap_callback(jsonnet_destroy_closure())?, @@ -32,9 +57,143 @@ impl JSApiSet for Jsonnet { } } +#[derive(jrsonnet_gcmodule::Trace)] +pub struct NativeJSCallback(String); + +impl NativeCallbackHandler for NativeJSCallback { + fn call( + &self, + args: &[jrsonnet_evaluator::Val], + ) -> jrsonnet_evaluator::Result { + debug!("NativeJSCallback called: {:?}", self.0); + let super_context = **super::CONTEXT.get().unwrap(); + let global = super_context + .global_object() + .expect("Unable to get super context"); + let func_map = global + .get_property("__jsonnet_func_map") + .expect("Unable to get global object"); + let func = func_map + .get_property(self.0.clone()) + .expect("Unable to get property"); + // debug!( + // "func: {:?}", + // from_qjs_value(func).expect("Unable to convert map ref to map") + // ); + let result; + if args.len() > 0 { + let args_str = serde_json::to_string(args).expect("Error converting args to JSON"); + let args_str = JSValue::String(args_str); + debug!( + "Calling function: {} with args = {}", + self.0, + args_str.to_string() + ); + result = func + .call( + &to_qjs_value(super_context, &JSValue::Undefined) + .expect("Unable to convert undefined"), + &[to_qjs_value(super_context, &args_str) + .expect("Unable to convert string to qjs value")], + ) + .expect("Unable to call function"); + // let result = from_qjs_value(result).expect("Unable to convert qjs value to value"); + // debug!("Result of calling JS function: {}", result.as_str().unwrap()); + } else { + let emtpy_str = JSValue::String("".to_string()); + let context = **super::CONTEXT.get().unwrap(); + result = func + .call( + &to_qjs_value(context, &JSValue::Undefined) + .expect("Unable to convert undefined"), + &[to_qjs_value(context, &emtpy_str) + .expect("Unable to convert string to qjs value")], + ) + .expect("Unable to call function"); + // let result = from_qjs_value(result).expect("Unable to convert qjs value to value"); + } + if result.is_object() { + debug!("Result is object"); + let constructor = result + .get_property("constructor") + .expect("Unable to get constructor"); + if !constructor.is_null_or_undefined() { + let constructor_name = constructor + .get_property("name") + .expect("Unable to find name in constructor") + .to_string(); + if constructor_name == "Promise" { + let resolved_result: Arc>> = Arc::new(Mutex::new(None)); + let resolved_error: Arc>> = Arc::new(Mutex::new(None)); + let then_func = result + .get_property("then") + .expect("Unable to find then on promise"); + if then_func.is_function() { + let resolved_result = resolved_result.clone(); + let resolved_error = resolved_error.clone(); + then_func + .call( + &result, + &[ + super_context + .wrap_callback(move |context, _this, args| { + resolved_result + .lock() + .unwrap() + .replace(args.get(0).unwrap().to_string()); + Ok(JSValue::Undefined) + }) + .expect("unable to wrap callback"), + super_context + .wrap_callback(move |context, _this, args| { + // resolvedError.replace(Some(args.get(0).unwrap().to_string())); + resolved_error + .lock() + .unwrap() + .replace(args.get(0).unwrap().to_string()); + Ok(JSValue::Undefined) + }) + .expect("Unable to wrap callback"), + ], + ) + .expect("Unable to call then function"); + super_context + .execute_pending() + .expect("Unable to execute pending tasks"); + } else { + panic!("then is not a function"); + } + + let result = resolved_result.lock().unwrap(); + let error = resolved_error.lock().unwrap(); + if result.is_some() { + Ok(Val::Str(result.as_ref().unwrap().into())) + } else { + Ok(Val::Str(error.as_ref().unwrap().into())) + } + } else { + Ok(Val::Str( + "Unable to find constructor property of returned type from function".into(), + )) + } + } else { + Ok(Val::Str("Result is an object but retuned object does not contain constructor function".into())) + } + } else if result.is_str() { + Ok(Val::Str(result.as_str().unwrap().into())) + } else { + // debug!("Result is unknown"); + Ok(Val::Str("Function does not return any result or promise".into())) + } + } +} + fn jsonnet_make_closure( ) -> impl FnMut(&JSContextRef, JSValueRef, &[JSValueRef]) -> anyhow::Result { - move |_ctx, _this, args| Ok(JSValue::Float(edgechains::jsonnet::jsonnet_make() as f64)) + move |_ctx, _this, args| { + let ptr = arakoo_jsonnet::jsonnet_make(); + Ok(JSValue::from(ptr as u64 as f64)) + } } fn jsonnet_ext_string_closure( @@ -47,10 +206,12 @@ fn jsonnet_ext_string_closure( args.len() - 1 )); } - let vm = args.get(0).unwrap().as_f64().unwrap(); + let vm = args.get(0).unwrap().as_f64()?; let key = args.get(1).unwrap().to_string(); let value = args.get(2).unwrap().to_string(); - edgechains::jsonnet::jsonnet_ext_string(vm as u64, &key, &value); + // edgechains::jsonnet::jsonnet_ext_string(vm as u64, &key, &value); + let ptr = vm as u64; + arakoo_jsonnet::jsonnet_ext_string(ptr as *mut arakoo_jsonnet::VM, &key, &value); Ok(JSValue::Undefined) } } @@ -62,10 +223,16 @@ fn jsonnet_evaluate_snippet_closure( if args.len() != 2 { return Err(anyhow::anyhow!("Expected 2 arguments, got {}", args.len())); } - let vm = args.get(0).unwrap().as_f64().unwrap(); + let vm = args.get(0).unwrap().as_f64()?; let code = args.get(1).unwrap().to_string(); let code = code.as_str(); - let out = edgechains::jsonnet::jsonnet_evaluate_snippet(vm as u64, "snippet", code); + // let out = edgechains::jsonnet::jsonnet_evaluate_snippet(vm as u64, "snippet", code); + let out = arakoo_jsonnet::jsonnet_evaluate_snippet( + vm as u64 as *mut arakoo_jsonnet::VM, + "snippet", + code, + ); + debug!("Result of evaluating snippet: {}", out.to_string()); Ok(out.into()) } } @@ -77,14 +244,45 @@ fn jsonnet_evaluate_file_closure( if args.len() != 2 { return Err(anyhow::anyhow!("Expected 2 arguments, got {}", args.len())); } - let vm = args.get(0).unwrap().as_f64().unwrap(); + let vm = args.get(0).unwrap().as_f64()?; let path = args.get(1).unwrap().to_string(); - let path = path.as_str(); - let out = edgechains::jsonnet::jsonnet_evaluate_file(vm as u64, path); + let code = edgechains::utils::read_file(path.as_str()); + let out = arakoo_jsonnet::jsonnet_evaluate_snippet( + vm as u64 as *mut arakoo_jsonnet::VM, + "snippet", + &code, + ); Ok(out.into()) } } +fn jsonnet_register_func_closure( +) -> impl FnMut(&JSContextRef, JSValueRef, &[JSValueRef]) -> anyhow::Result { + move |_ctx, _this, args| { + // check the number of arguments + if args.len() != 3 { + return Err(anyhow::anyhow!("Expected 3 arguments, got {}", args.len())); + } + let vm = args.get(0).unwrap().as_f64().unwrap(); + let func_name = args.get(1).unwrap().to_string(); + let args_num = args.get(2).unwrap().as_f64().unwrap(); + // edgechains::jsonnet::jsonnet_register_func(vm as u64, &func_name, args_num as u32); + let vm = unsafe { &*(vm as u64 as *mut arakoo_jsonnet::VM) }; + let any_resolver = vm.state.context_initializer(); + let args_vec = vec![String::from("x"); args_num as usize]; + any_resolver + .as_any() + .downcast_ref::() + .expect("only arakoo context initializer supported") + .add_native( + func_name.clone(), + NativeCallback::new(args_vec, NativeJSCallback(func_name.clone())), + ); + debug!("Registered function: {}", func_name); + Ok(JSValue::Undefined) + } +} + fn jsonnet_destroy_closure( ) -> impl FnMut(&JSContextRef, JSValueRef, &[JSValueRef]) -> anyhow::Result { move |_ctx, _this, args| { @@ -92,8 +290,8 @@ fn jsonnet_destroy_closure( if args.len() != 1 { return Err(anyhow::anyhow!("Expected 1 arguments, got {}", args.len())); } - let vm = args.get(0).unwrap().as_f64().unwrap(); - edgechains::jsonnet::jsonnet_destroy(vm as u64); + let vm = args.get(0).unwrap().as_f64()?; + arakoo_jsonnet::jsonnet_destroy(vm as u64 as *mut arakoo_jsonnet::VM); Ok(JSValue::Undefined) } } diff --git a/JS/wasm/crates/arakoo-core/src/apis/mod.rs b/JS/wasm/crates/arakoo-core/src/apis/mod.rs index 359481077..8482d434b 100644 --- a/JS/wasm/crates/arakoo-core/src/apis/mod.rs +++ b/JS/wasm/crates/arakoo-core/src/apis/mod.rs @@ -50,6 +50,8 @@ pub use api_config::APIConfig; pub use console::LogStream; pub use runtime_ext::RuntimeExt; +use super::CONTEXT; + pub mod http; pub mod types; diff --git a/JS/wasm/crates/arakoo-core/src/lib.rs b/JS/wasm/crates/arakoo-core/src/lib.rs index 53f36b425..b81761022 100644 --- a/JS/wasm/crates/arakoo-core/src/lib.rs +++ b/JS/wasm/crates/arakoo-core/src/lib.rs @@ -71,7 +71,7 @@ static mut RUNTIME_INSTANCE: Option = None; // fn on_reject(context: &JSContextRef, _this: JSValueRef, args: &[JSValueRef]) -> Result { // // (*args).clone_into(&mut cloned_args); // let mut qjs_value = Option::None; -// if (args.len() > 0) { +// if args.len() > 0 { // for arg in args { // qjs_value = Some(from_qjs_value(*arg).unwrap()); // println!("Arg reject : {:?}", qjs_value.as_ref().unwrap()); diff --git a/JS/wasm/crates/serve/src/binding.rs b/JS/wasm/crates/serve/src/binding.rs index 3752d28cb..b4be01074 100644 --- a/JS/wasm/crates/serve/src/binding.rs +++ b/JS/wasm/crates/serve/src/binding.rs @@ -11,7 +11,9 @@ use reqwest::Url; // use arakoo_jsonnet::{ // ext_string, jsonnet_destroy, jsonnet_evaluate_file, jsonnet_evaluate_snippet, jsonnet_make, // }; -use jrsonnet_evaluator::{function::TlaArg, gc::GcHashMap, manifest::ManifestFormat, trace::TraceFormat, State}; +use jrsonnet_evaluator::{ + function::TlaArg, gc::GcHashMap, manifest::ManifestFormat, trace::TraceFormat, State, +}; use jrsonnet_parser::IStr; use std::{fs, io}; @@ -19,43 +21,18 @@ use std::{fs, io}; use tracing::error; // use wasmtime::*; -use crate::io::{WasmInput, WasmOutput}; - #[async_trait] -impl super::jsonnet::Host for super::Host{ - async fn jsonnet_make(&mut self,) -> wasmtime::Result { - let ptr = arakoo_jsonnet::jsonnet_make(); - Ok(ptr as u64) - } - - async fn jsonnet_evaluate_snippet(&mut self, vm: u64,file:String, code: String) -> wasmtime::Result { - let out = arakoo_jsonnet::jsonnet_evaluate_snippet(vm as *mut arakoo_jsonnet::VM, &file, &code); - Ok(out) - } - - async fn jsonnet_evaluate_file(&mut self, vm: u64, path: String) -> wasmtime::Result { +impl super::utils::Host for super::Host { + async fn read_file(&mut self, path: String) -> wasmtime::Result { let code = fs::read_to_string(&path).map_err(|e| { error!("Failed to read file {}: {}", path, e); io::Error::new(io::ErrorKind::Other, e) })?; - let out = arakoo_jsonnet::jsonnet_evaluate_snippet(vm as *mut arakoo_jsonnet::VM, "snippet", &code); - Ok(out) - } - - async fn jsonnet_ext_string(&mut self, vm: u64, key: String, value: String) -> wasmtime::Result<()> { - arakoo_jsonnet::jsonnet_ext_string(vm as *mut arakoo_jsonnet::VM, &key, &value); - Ok(()) - } - - async fn jsonnet_destroy(&mut self, vm: u64) -> wasmtime::Result<()> { - arakoo_jsonnet::jsonnet_destroy(vm as *mut arakoo_jsonnet::VM); - Ok(()) + Ok(code) } } -// Bindings for jsonnet - #[async_trait] impl super::outbound_http::Host for super::Host { async fn send_request( @@ -68,8 +45,7 @@ impl super::outbound_http::Host for super::Host { let method = method_from(req.method); let url = Url::parse(&req.uri).map_err(|_| HttpError::InvalidUrl)?; - let headers = - request_headers(req.headers).map_err(|_| HttpError::RuntimeError)?; + let headers = request_headers(req.headers).map_err(|_| HttpError::RuntimeError)?; let body = req.body.unwrap_or_default().to_vec(); if !req.params.is_empty() { diff --git a/JS/wasm/crates/serve/src/lib.rs b/JS/wasm/crates/serve/src/lib.rs index 716cc3812..97ca8e021 100644 --- a/JS/wasm/crates/serve/src/lib.rs +++ b/JS/wasm/crates/serve/src/lib.rs @@ -1,7 +1,6 @@ // mod binding; use wit::arakoo::edgechains::http as outbound_http; use wit::arakoo::edgechains::http_types::HttpError; -use wit::arakoo::edgechains::jsonnet; mod binding; mod io; @@ -40,6 +39,7 @@ use wasmtime::component::{Component, Linker}; use wasmtime::{Config, Engine, Store, WasmBacktraceDetails}; use wit::arakoo::edgechains::http_types; use wit::exports::arakoo::edgechains::inbound_http::{self}; +use wit::arakoo::edgechains::utils; use crate::{ // binding::add_jsonnet_to_linker, diff --git a/JS/wasm/examples/ec-wasmjs-hono/build.js b/JS/wasm/examples/ec-wasmjs-hono/build.js index 575c41cb3..2b7a8559b 100644 --- a/JS/wasm/examples/ec-wasmjs-hono/build.js +++ b/JS/wasm/examples/ec-wasmjs-hono/build.js @@ -34,7 +34,7 @@ build({ pattern: [["export default", "_export = "]], }), ], - format: "esm", + format: "cjs", target: "esnext", platform: "node", // external: ["arakoo"], diff --git a/JS/wasm/examples/ec-wasmjs-hono/src/index.js b/JS/wasm/examples/ec-wasmjs-hono/src/index.js index e4a22e8fc..1a67c4335 100644 --- a/JS/wasm/examples/ec-wasmjs-hono/src/index.js +++ b/JS/wasm/examples/ec-wasmjs-hono/src/index.js @@ -6,6 +6,11 @@ import Jsonnet from "@arakoodev/jsonnet"; let jsonnet = new Jsonnet(); const app = new Hono(); + +function greet() { + return "Hello from JS"; +} + const env = {}; app.get("/hello", (c) => { @@ -27,6 +32,64 @@ app.get("/", (c) => { return c.json(JSON.parse(result)); }); +app.get("/func", (c) => { + const code = ` + local username = std.extVar('name'); + local Person(name='Alice') = { + name: name, + welcome: 'Hello ' + name + '!', + }; + { + person1: Person(username), + person2: Person('Bob'), + result : arakoo.native("greet")() + }`; + let result = jsonnet.extString("name", "ll").javascriptCallback("greet", greet).evaluateSnippet(code); + return c.json(JSON.parse(result)); +}); + +app.get("/async-func/:id", async (c) => { + let id = c.req.param("id"); + async function asyncGetAtodo(id) { + try { + let response = await fetch(`https://jsonplaceholder.typicode.com/todos/${id}`); + let body = await response.json(); + return JSON.stringify(body); + } catch (error) { + console.log("error occured"); + console.log(error); + return c.json(output); + } + } + + let result = jsonnet.extString("id", id).javascriptCallback("getAtodo", asyncGetAtodo).evaluateSnippet(` + local todo = std.parseJson(arakoo.native("getAtodo")(std.extVar("id"))); + { + result : todo.title + }`); + return c.json(JSON.parse(result)); +}); + +app.get("/add", (c) => { + function add(arg1, arg2, arg3) { + console.log("Args recieved: ", arg1, arg2, arg3); + return arg1 + arg2 + arg3; + } + const code = ` + local username = std.extVar('name'); + local Person(name='Alice') = { + name: name, + welcome: 'Hello ' + name + '!', + }; + { + person1: Person(username), + person2: Person('Bob'), + result : arakoo.native("add")(1,2,3) + }`; + let result = jsonnet.extString("name", "ll").javascriptCallback("add", add).evaluateSnippet(code); + return c.json(JSON.parse(result)); +}); + app.get("/file", (c) => { try { let result = jsonnet @@ -76,3 +139,4 @@ app.notFound((c) => { }); app.fire(); +// globalThis._export = app; diff --git a/JS/wasm/wit/arakoo.wit b/JS/wasm/wit/arakoo.wit index a4433c5f2..e60f95e5e 100644 --- a/JS/wasm/wit/arakoo.wit +++ b/JS/wasm/wit/arakoo.wit @@ -2,6 +2,6 @@ package arakoo:edgechains; world reactor { import http; - import jsonnet; + import utils; export inbound-http; } \ No newline at end of file diff --git a/JS/wasm/wit/jsonnet.wit b/JS/wasm/wit/jsonnet.wit deleted file mode 100644 index dd9e200f8..000000000 --- a/JS/wasm/wit/jsonnet.wit +++ /dev/null @@ -1,10 +0,0 @@ -interface jsonnet { - record vars { - key:string - } - jsonnet-make: func() -> u64; - jsonnet-evaluate-snippet: func(vm: u64,file: string,code: string) -> string; - jsonnet-evaluate-file: func(vm: u64,path: string) -> string; - jsonnet-ext-string: func(vm: u64,key: string, value: string); - jsonnet-destroy: func(vm: u64); -} diff --git a/JS/wasm/wit/utils.wit b/JS/wasm/wit/utils.wit new file mode 100644 index 000000000..cc6db19d7 --- /dev/null +++ b/JS/wasm/wit/utils.wit @@ -0,0 +1,3 @@ +interface utils { + read-file: func(path: string) -> string; +}