diff --git a/Makefile b/Makefile index b4516e64581..34e3630928c 100644 --- a/Makefile +++ b/Makefile @@ -91,6 +91,11 @@ test-libsql-sqlite: dev-libsql-sqlite test-qe-st test-driver-adapter-libsql: test-libsql-sqlite +dev-libsql-sqlite-wasm: build-qe-napi build-connector-kit-js + cp $(CONFIG_PATH)/libsql-sqlite-wasm $(CONFIG_FILE) + +test-libsql-sqlite-wasm: dev-libsql-sqlite-wasm test-qe-st + start-postgres9: docker compose -f docker-compose.yml up --wait -d --remove-orphans postgres9 @@ -128,6 +133,11 @@ dev-pg-postgres13: start-pg-postgres13 test-pg-postgres13: dev-pg-postgres13 test-qe-st +dev-pg-postgres13-wasm: start-pg-postgres13 + cp $(CONFIG_PATH)/pg-postgres13-wasm $(CONFIG_FILE) + +test-pg-postgres13-wasm: dev-pg-postgres13-wasm test-qe-st + test-driver-adapter-pg: test-pg-postgres13 start-neon-postgres13: @@ -138,6 +148,11 @@ dev-neon-ws-postgres13: start-neon-postgres13 build-qe-napi build-connector-kit- test-neon-ws-postgres13: dev-neon-ws-postgres13 test-qe-st +dev-neon-ws-postgres13-wasm: start-neon-postgres13 build-qe-napi build-connector-kit-js + cp $(CONFIG_PATH)/neon-ws-postgres13-wasm $(CONFIG_FILE) + +test-neon-ws-postgres13-wasm: dev-neon-ws-postgres13-wasm test-qe-st + test-driver-adapter-neon: test-neon-ws-postgres13 start-postgres14: @@ -270,6 +285,11 @@ dev-planetscale-vitess8: start-planetscale-vitess8 build-qe-napi build-connector test-planetscale-vitess8: dev-planetscale-vitess8 test-qe-st +dev-planetscale-vitess8-wasm: start-planetscale-vitess8 build-qe-napi build-connector-kit-js + cp $(CONFIG_PATH)/planetscale-vitess8-wasm $(CONFIG_FILE) + +test-planetscale-vitess8-wasm: dev-planetscale-vitess8-wasm test-qe-st + test-driver-adapter-planetscale: test-planetscale-vitess8 ###################### diff --git a/query-engine/connector-test-kit-rs/query-tests-setup/src/config.rs b/query-engine/connector-test-kit-rs/query-tests-setup/src/config.rs index 4af4e763298..751e0dd23c2 100644 --- a/query-engine/connector-test-kit-rs/query-tests-setup/src/config.rs +++ b/query-engine/connector-test-kit-rs/query-tests-setup/src/config.rs @@ -3,10 +3,26 @@ use crate::{ PostgresConnectorTag, SqlServerConnectorTag, SqliteConnectorTag, TestResult, VitessConnectorTag, }; use serde::Deserialize; -use std::{convert::TryFrom, env, fs::File, io::Read, path::PathBuf}; +use std::{convert::TryFrom, env, fmt::Display, fs::File, io::Read, path::PathBuf}; static TEST_CONFIG_FILE_NAME: &str = ".test_config"; +#[derive(Debug, Default, Deserialize, Clone)] +enum TestExecutorEngine { + #[default] + NAPI, + WASM, +} + +impl Display for TestExecutorEngine { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TestExecutorEngine::NAPI => f.write_str("NAPI"), + TestExecutorEngine::WASM => f.write_str("WASM"), + } + } +} + /// The central test configuration. #[derive(Debug, Default, Deserialize)] pub struct TestConfig { @@ -27,6 +43,11 @@ pub struct TestConfig { /// Env key: `EXTERNAL_TEST_EXECUTOR` external_test_executor: Option, + /// Specifies which engine to use within external test executor. + /// If omitted, NAPI engine will be used. + /// Env key: `EXTERNAL_TEST_EXECUTOR_ENGINE` + external_test_executor_engine: Option, + /// The driver adapter to use when running tests, will be forwarded to the external test /// executor by setting the `DRIVER_ADAPTER` env var when spawning the executor process driver_adapter: Option, @@ -109,6 +130,7 @@ impl TestConfig { println!("* CI? {}", self.is_ci); if self.external_test_executor.as_ref().is_some() { println!("* External test executor: {}", self.external_test_executor().unwrap_or_default()); + println!("* External test executor engine: {:?}", self.external_test_executor().unwrap_or_default()); println!("* Driver adapter: {}", self.driver_adapter().unwrap_or_default()); println!("* Driver adapter url override: {}", self.json_stringify_driver_adapter_config()); } @@ -119,6 +141,10 @@ impl TestConfig { let connector = std::env::var("TEST_CONNECTOR").ok(); let connector_version = std::env::var("TEST_CONNECTOR_VERSION").ok(); let external_test_executor = std::env::var("EXTERNAL_TEST_EXECUTOR").ok(); + let external_test_executor_engine = std::env::var("EXTERNAL_TEST_EXECUTOR_ENGINE") + .map(|value| serde_json::from_str::(&value).ok()) + .unwrap_or_default(); + let driver_adapter = std::env::var("DRIVER_ADAPTER").ok(); let driver_adapter_config = std::env::var("DRIVER_ADAPTER_CONFIG") .map(|config| serde_json::from_str::(config.as_str()).ok()) @@ -134,6 +160,7 @@ impl TestConfig { external_test_executor, driver_adapter, driver_adapter_config, + external_test_executor_engine, }) } @@ -234,6 +261,12 @@ impl TestConfig { ); } + if self.external_test_executor_engine.is_some() && self.external_test_executor.is_none() { + exit_with_message( + "External test executor engine can be used only if EXTERNAL_TEST_EXECUTOR env var is set.", + ); + } + if self.driver_adapter.is_some() && self.external_test_executor.is_none() { exit_with_message( "When using a driver adapter, the external test executor (EXTERNAL_TEST_EXECUTOR env var) must be set.", @@ -294,11 +327,16 @@ impl TestConfig { vec!( ( "DRIVER_ADAPTER".to_string(), - self.driver_adapter.clone().unwrap_or_default()), + self.driver_adapter.clone().unwrap_or_default() + ), ( "DRIVER_ADAPTER_CONFIG".to_string(), self.json_stringify_driver_adapter_config() ), + ( + "EXTERNAL_TEST_EXECUTOR_ENGINE".to_string(), + self.external_test_executor_engine.clone().unwrap_or_default().to_string(), + ), ( "PRISMA_DISABLE_QUAINT_EXECUTORS".to_string(), "1".to_string(), diff --git a/query-engine/connector-test-kit-rs/query-tests-setup/src/connector_tag/js/external_process.rs b/query-engine/connector-test-kit-rs/query-tests-setup/src/connector_tag/js/external_process.rs index 583d5058c62..265f535454a 100644 --- a/query-engine/connector-test-kit-rs/query-tests-setup/src/connector_tag/js/external_process.rs +++ b/query-engine/connector-test-kit-rs/query-tests-setup/src/connector_tag/js/external_process.rs @@ -74,7 +74,7 @@ impl ExecutorProcess { }; self.task_handle.send((method_call, sender)).await?; - let raw_response = receiver.await?; + let raw_response = receiver.await??; tracing::debug!(%raw_response); let response = serde_json::from_value(raw_response)?; Ok(response) @@ -91,7 +91,10 @@ pub(super) static EXTERNAL_PROCESS: Lazy = } }); -type ReqImpl = (jsonrpc_core::MethodCall, oneshot::Sender); +type ReqImpl = ( + jsonrpc_core::MethodCall, + oneshot::Sender>, +); fn start_rpc_thread(mut receiver: mpsc::Receiver) -> Result<()> { use std::process::Stdio; @@ -119,7 +122,7 @@ fn start_rpc_thread(mut receiver: mpsc::Receiver) -> Result<()> { let mut stdout = BufReader::new(process.stdout.unwrap()).lines(); let mut stdin = process.stdin.unwrap(); - let mut pending_requests: HashMap> = + let mut pending_requests: HashMap>> = HashMap::new(); loop { @@ -140,10 +143,11 @@ fn start_rpc_thread(mut receiver: mpsc::Receiver) -> Result<()> { // The other end may be dropped if the whole // request future was dropped and not polled to // completion, so we ignore send errors here. - _ = sender.send(success.result); + _ = sender.send(Ok(success.result)); } jsonrpc_core::Output::Failure(err) => { - panic!("error response from jsonrpc: {err:?}") + tracing::error!("error response from jsonrpc: {err:?}"); + _ = sender.send(Err(Box::new(err.error))); } } } diff --git a/query-engine/connector-test-kit-rs/test-configs/libsql-sqlite b/query-engine/connector-test-kit-rs/test-configs/libsql-sqlite index 9638e3a2284..4d24cd29e41 100644 --- a/query-engine/connector-test-kit-rs/test-configs/libsql-sqlite +++ b/query-engine/connector-test-kit-rs/test-configs/libsql-sqlite @@ -1,5 +1,6 @@ { "connector": "sqlite", "driver_adapter": "libsql", - "external_test_executor": "default" + "external_test_executor": "default", + "external_test_executor_engine": "NAPI" } \ No newline at end of file diff --git a/query-engine/connector-test-kit-rs/test-configs/libsql-sqlite-wasm b/query-engine/connector-test-kit-rs/test-configs/libsql-sqlite-wasm new file mode 100644 index 00000000000..32ef323e444 --- /dev/null +++ b/query-engine/connector-test-kit-rs/test-configs/libsql-sqlite-wasm @@ -0,0 +1,6 @@ +{ + "connector": "sqlite", + "driver_adapter": "libsql", + "external_test_executor": "default", + "external_test_executor_engine": "WASM" +} \ No newline at end of file diff --git a/query-engine/connector-test-kit-rs/test-configs/neon-ws-postgres13 b/query-engine/connector-test-kit-rs/test-configs/neon-ws-postgres13 index 0097d8c91f5..c96a593dc19 100644 --- a/query-engine/connector-test-kit-rs/test-configs/neon-ws-postgres13 +++ b/query-engine/connector-test-kit-rs/test-configs/neon-ws-postgres13 @@ -3,5 +3,6 @@ "version": "13", "driver_adapter": "neon:ws", "driver_adapter_config": { "proxyUrl": "127.0.0.1:5488/v1" }, - "external_test_executor": "default" + "external_test_executor": "default", + "external_test_executor_engine": "NAPI" } \ No newline at end of file diff --git a/query-engine/connector-test-kit-rs/test-configs/neon-ws-postgres13-wasm b/query-engine/connector-test-kit-rs/test-configs/neon-ws-postgres13-wasm new file mode 100644 index 00000000000..fcc42e0db7a --- /dev/null +++ b/query-engine/connector-test-kit-rs/test-configs/neon-ws-postgres13-wasm @@ -0,0 +1,8 @@ +{ + "connector": "postgres", + "version": "13", + "driver_adapter": "neon:ws", + "driver_adapter_config": { "proxyUrl": "127.0.0.1:5488/v1" }, + "external_test_executor": "default", + "external_test_executor_engine": "WASM" +} \ No newline at end of file diff --git a/query-engine/connector-test-kit-rs/test-configs/pg-postgres13 b/query-engine/connector-test-kit-rs/test-configs/pg-postgres13 index 00f0c75ed73..b5d4c9faf82 100644 --- a/query-engine/connector-test-kit-rs/test-configs/pg-postgres13 +++ b/query-engine/connector-test-kit-rs/test-configs/pg-postgres13 @@ -2,5 +2,6 @@ "connector": "postgres", "version": "13", "driver_adapter": "pg", - "external_test_executor": "default" + "external_test_executor": "default", + "external_test_executor_engine": "NAPI" } \ No newline at end of file diff --git a/query-engine/connector-test-kit-rs/test-configs/pg-postgres13-wasm b/query-engine/connector-test-kit-rs/test-configs/pg-postgres13-wasm new file mode 100644 index 00000000000..33999aadbde --- /dev/null +++ b/query-engine/connector-test-kit-rs/test-configs/pg-postgres13-wasm @@ -0,0 +1,7 @@ +{ + "connector": "postgres", + "version": "13", + "driver_adapter": "pg", + "external_test_executor": "default", + "external_test_executor_engine": "WASM" +} \ No newline at end of file diff --git a/query-engine/connector-test-kit-rs/test-configs/planetscale-vitess8 b/query-engine/connector-test-kit-rs/test-configs/planetscale-vitess8 index 48c89c79427..012d898d86b 100644 --- a/query-engine/connector-test-kit-rs/test-configs/planetscale-vitess8 +++ b/query-engine/connector-test-kit-rs/test-configs/planetscale-vitess8 @@ -3,5 +3,6 @@ "version": "8.0", "driver_adapter": "planetscale", "driver_adapter_config": { "proxyUrl": "http://root:root@127.0.0.1:8085" }, - "external_test_executor": "default" + "external_test_executor": "default", + "external_test_executor_engine": "NAPI" } diff --git a/query-engine/connector-test-kit-rs/test-configs/planetscale-vitess8-wasm b/query-engine/connector-test-kit-rs/test-configs/planetscale-vitess8-wasm new file mode 100644 index 00000000000..75080f7386e --- /dev/null +++ b/query-engine/connector-test-kit-rs/test-configs/planetscale-vitess8-wasm @@ -0,0 +1,8 @@ +{ + "connector": "vitess", + "version": "8.0", + "driver_adapter": "planetscale", + "driver_adapter_config": { "proxyUrl": "http://root:root@127.0.0.1:8085" }, + "external_test_executor": "default", + "external_test_executor_engine": "WASM" +} diff --git a/query-engine/driver-adapters/connector-test-kit-executor/src/index.ts b/query-engine/driver-adapters/connector-test-kit-executor/src/index.ts index 2318c052576..d84382b7cd2 100644 --- a/query-engine/driver-adapters/connector-test-kit-executor/src/index.ts +++ b/query-engine/driver-adapters/connector-test-kit-executor/src/index.ts @@ -1,5 +1,4 @@ import * as qe from './qe' -import * as engines from './engines/Library' import * as readline from 'node:readline' import * as jsonRpc from './jsonRpc' @@ -76,7 +75,7 @@ async function main(): Promise { } const state: Record = {} @@ -215,7 +214,7 @@ function respondOk(requestId: number, payload: unknown) { console.log(JSON.stringify(msg)) } -async function initQe(url: string, prismaSchema: string, logCallback: qe.QueryLogCallback): Promise<[engines.QueryEngineInstance, ErrorCapturingDriverAdapter]> { +async function initQe(url: string, prismaSchema: string, logCallback: qe.QueryLogCallback): Promise<[qe.QueryEngine, ErrorCapturingDriverAdapter]> { const adapter = await adapterFromEnv(url) as DriverAdapter const errorCapturingAdapter = bindAdapter(adapter) const engineInstance = qe.initQueryEngine(errorCapturingAdapter, prismaSchema, logCallback, debug) diff --git a/query-engine/driver-adapters/connector-test-kit-executor/src/qe.ts b/query-engine/driver-adapters/connector-test-kit-executor/src/qe.ts index 186d7a9e80d..109cdf635dc 100644 --- a/query-engine/driver-adapters/connector-test-kit-executor/src/qe.ts +++ b/query-engine/driver-adapters/connector-test-kit-executor/src/qe.ts @@ -1,22 +1,25 @@ import type { ErrorCapturingDriverAdapter } from '@prisma/driver-adapter-utils' -import * as lib from './engines/Library' +import { WasmQueryEngine } from './wasm' +import * as napi from './engines/Library' import * as os from 'node:os' import * as path from 'node:path' +import { fileURLToPath } from 'node:url' -export type QueryLogCallback = (log: string) => void +const dirname = path.dirname(fileURLToPath(import.meta.url)) -export function initQueryEngine(adapter: ErrorCapturingDriverAdapter, datamodel: string, queryLogCallback: QueryLogCallback, debug: (...args: any[]) => void): lib.QueryEngineInstance { - // I assume nobody will run this on Windows ¯\_(ツ)_/¯ - const libExt = os.platform() === 'darwin' ? 'dylib' : 'so' - const dirname = path.dirname(new URL(import.meta.url).pathname) +export interface QueryEngine { + connect(trace: string): Promise + disconnect(trace: string): Promise; + query(body: string, trace: string, tx_id?: string): Promise; + startTransaction(input: string, trace: string): Promise; + commitTransaction(tx_id: string, trace: string): Promise; + rollbackTransaction(tx_id: string, trace: string): Promise; +} - const libQueryEnginePath = path.join(dirname, `../../../../target/debug/libquery_engine.${libExt}`) +export type QueryLogCallback = (log: string) => void - const libqueryEngine = { exports: {} as unknown as lib.Library } - // @ts-ignore - process.dlopen(libqueryEngine, libQueryEnginePath) - const QueryEngine = libqueryEngine.exports.QueryEngine +export function initQueryEngine(adapter: ErrorCapturingDriverAdapter, datamodel: string, queryLogCallback: QueryLogCallback, debug: (...args: any[]) => void): QueryEngine { const queryEngineOptions = { datamodel, @@ -37,5 +40,28 @@ export function initQueryEngine(adapter: ErrorCapturingDriverAdapter, datamodel: debug(parsed) } - return new QueryEngine(queryEngineOptions, logCallback, adapter) + const engineFromEnv = process.env.EXTERNAL_TEST_EXECUTOR_ENGINE ?? 'napi' + if (engineFromEnv === 'WASM') { + return new WasmQueryEngine(queryEngineOptions, logCallback, adapter) + } else if (engineFromEnv === 'NAPI') { + const { QueryEngine } = loadNapiEngine() + return new QueryEngine(queryEngineOptions, logCallback, adapter) + } else { + throw new TypeError(`Invalid EXTERNAL_TEST_EXECUTOR_ENGINE value: ${engineFromEnv}. Expected NAPI or WASM`) + } + + } + +function loadNapiEngine(): napi.Library { + // I assume nobody will run this on Windows ¯\_(ツ)_/¯ + const libExt = os.platform() === 'darwin' ? 'dylib' : 'so' + + const libQueryEnginePath = path.join(dirname, `../../../../target/debug/libquery_engine.${libExt}`) + + const libqueryEngine = { exports: {} as unknown as napi.Library } + // @ts-ignore + process.dlopen(libqueryEngine, libQueryEnginePath) + + return libqueryEngine.exports +} \ No newline at end of file diff --git a/query-engine/driver-adapters/connector-test-kit-executor/src/wasm.ts b/query-engine/driver-adapters/connector-test-kit-executor/src/wasm.ts new file mode 100644 index 00000000000..439fd0c3f94 --- /dev/null +++ b/query-engine/driver-adapters/connector-test-kit-executor/src/wasm.ts @@ -0,0 +1,14 @@ +import * as wasm from '../../../query-engine-wasm/pkg/query_engine_bg.js' +import fs from 'node:fs/promises' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +const dirname = path.dirname(fileURLToPath(import.meta.url)) + +const bytes = await fs.readFile(path.resolve(dirname, '..', '..', '..', 'query-engine-wasm', 'pkg', 'query_engine_bg.wasm')) +const module = new WebAssembly.Module(bytes) +const instance = new WebAssembly.Instance(module, { './query_engine_bg.js': wasm }) +wasm.__wbg_set_wasm(instance.exports); +wasm.init() + +export const WasmQueryEngine = wasm.QueryEngine \ No newline at end of file diff --git a/query-engine/driver-adapters/connector-test-kit-executor/tsconfig.json b/query-engine/driver-adapters/connector-test-kit-executor/tsconfig.json index 516c114b3e1..20fc4bd62ff 100644 --- a/query-engine/driver-adapters/connector-test-kit-executor/tsconfig.json +++ b/query-engine/driver-adapters/connector-test-kit-executor/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { "target": "ES2022", "module": "ESNext", - "lib": ["ES2022"], + "lib": ["ES2022", "DOM"], "moduleResolution": "Bundler", "esModuleInterop": false, "isolatedModules": true, @@ -17,7 +17,7 @@ "skipDefaultLibCheck": true, "skipLibCheck": true, "emitDeclarationOnly": true, - "resolveJsonModule": true + "resolveJsonModule": true, }, "exclude": ["**/dist", "**/declaration", "**/node_modules", "**/src/__tests__"] } \ No newline at end of file