From 66a344faf542dc39d4c2d4f893b839a051fe35be Mon Sep 17 00:00:00 2001 From: Bo Lu Date: Wed, 23 Oct 2024 12:42:42 +1100 Subject: [PATCH 01/12] feat: add Interval and Bytea data types support --- supabase-wrappers/src/interface.rs | 32 ++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/supabase-wrappers/src/interface.rs b/supabase-wrappers/src/interface.rs index af6ca207..a53aa36d 100644 --- a/supabase-wrappers/src/interface.rs +++ b/supabase-wrappers/src/interface.rs @@ -4,11 +4,11 @@ use crate::instance::ForeignServer; use crate::FdwRoutine; use pgrx::pg_sys::panic::ErrorReport; -use pgrx::prelude::{Date, Timestamp, TimestampWithTimeZone}; +use pgrx::prelude::{Date, Interval, Timestamp, TimestampWithTimeZone}; use pgrx::{ datum::Uuid, fcinfo, - pg_sys::{self, BuiltinOid, Datum, Oid}, + pg_sys::{self, bytea, BuiltinOid, Datum, Oid}, AllocatedByRust, AnyNumeric, FromDatum, IntoDatum, JsonB, PgBuiltInOids, PgOid, }; use std::collections::HashMap; @@ -47,7 +47,9 @@ pub enum Cell { Date(Date), Timestamp(Timestamp), Timestamptz(TimestampWithTimeZone), + Interval(Interval), Json(JsonB), + Bytea(*mut bytea), Uuid(Uuid), BoolArray(Vec>), I16Array(Vec>), @@ -73,7 +75,9 @@ impl Clone for Cell { Cell::Date(v) => Cell::Date(*v), Cell::Timestamp(v) => Cell::Timestamp(*v), Cell::Timestamptz(v) => Cell::Timestamptz(*v), + Cell::Interval(v) => Cell::Interval(*v), Cell::Json(v) => Cell::Json(JsonB(v.0.clone())), + Cell::Bytea(v) => Cell::Bytea(*v), Cell::Uuid(v) => Cell::Uuid(*v), Cell::BoolArray(v) => Cell::BoolArray(v.clone()), Cell::I16Array(v) => Cell::I16Array(v.clone()), @@ -154,7 +158,21 @@ impl fmt::Display for Cell { .expect("timestamptz should be a valid string") ) }, + Cell::Interval(v) => write!(f, "{}", v), Cell::Json(v) => write!(f, "{:?}", v), + Cell::Bytea(v) => { + let byte_u8 = unsafe { pgrx::varlena::varlena_to_byte_slice(*v) }; + let hex = byte_u8 + .iter() + .map(|b| format!("{:02X}", b)) + .collect::>() + .join(""); + if hex.is_empty() { + write!(f, "''") + } else { + write!(f, r#"'\x{}'"#, hex) + } + } Cell::Uuid(v) => write!(f, "{}", v), Cell::BoolArray(v) => write_array(v, f), Cell::I16Array(v) => write_array(v, f), @@ -182,7 +200,9 @@ impl IntoDatum for Cell { Cell::Date(v) => v.into_datum(), Cell::Timestamp(v) => v.into_datum(), Cell::Timestamptz(v) => v.into_datum(), + Cell::Interval(v) => v.into_datum(), Cell::Json(v) => v.into_datum(), + Cell::Bytea(v) => Some(Datum::from(v)), Cell::Uuid(v) => v.into_datum(), Cell::BoolArray(v) => v.into_datum(), Cell::I16Array(v) => v.into_datum(), @@ -212,7 +232,9 @@ impl IntoDatum for Cell { || other == pg_sys::DATEOID || other == pg_sys::TIMESTAMPOID || other == pg_sys::TIMESTAMPTZOID + || other == pg_sys::INTERVALOID || other == pg_sys::JSONBOID + || other == pg_sys::BYTEAOID || other == pg_sys::UUIDOID || other == pg_sys::BOOLARRAYOID || other == pg_sys::INT2ARRAYOID @@ -265,9 +287,15 @@ impl FromDatum for Cell { PgOid::BuiltIn(PgBuiltInOids::TIMESTAMPTZOID) => { TimestampWithTimeZone::from_datum(datum, is_null).map(Cell::Timestamptz) } + PgOid::BuiltIn(PgBuiltInOids::INTERVALOID) => { + Interval::from_datum(datum, is_null).map(Cell::Interval) + } PgOid::BuiltIn(PgBuiltInOids::JSONBOID) => { JsonB::from_datum(datum, is_null).map(Cell::Json) } + PgOid::BuiltIn(PgBuiltInOids::BYTEAOID) => { + Some(Cell::Bytea(datum.cast_mut_ptr::())) + } PgOid::BuiltIn(PgBuiltInOids::UUIDOID) => { Uuid::from_datum(datum, is_null).map(Cell::Uuid) } From 3866d1058b56bf1ebff73438fc225d427c746efa Mon Sep 17 00:00:00 2001 From: Bo Lu Date: Wed, 23 Oct 2024 16:43:05 +1100 Subject: [PATCH 02/12] fix: wasm fdw test path --- wrappers/src/fdw/wasm_fdw/tests.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wrappers/src/fdw/wasm_fdw/tests.rs b/wrappers/src/fdw/wasm_fdw/tests.rs index 26220bbc..9fa944ec 100644 --- a/wrappers/src/fdw/wasm_fdw/tests.rs +++ b/wrappers/src/fdw/wasm_fdw/tests.rs @@ -19,7 +19,7 @@ mod tests { r#"CREATE SERVER snowflake_server FOREIGN DATA WRAPPER wasm_wrapper OPTIONS ( - fdw_package_url 'file:///home/runner/work/wrappers/wrappers/wasm-wrappers/fdw/snowflake_fdw/target/wasm32-unknown-unknown/release/snowflake_fdw.wasm', + fdw_package_url 'file://../../../wasm-wrappers/fdw/snowflake_fdw/target/wasm32-unknown-unknown/release/snowflake_fdw.wasm', fdw_package_name 'supabase:snowflake-fdw', fdw_package_version '>=0.1.0', api_url 'http://localhost:8096/snowflake/{}', @@ -64,7 +64,7 @@ mod tests { r#"CREATE SERVER paddle_server FOREIGN DATA WRAPPER wasm_wrapper OPTIONS ( - fdw_package_url 'file:///home/runner/work/wrappers/wrappers/wasm-wrappers/fdw/paddle_fdw/target/wasm32-unknown-unknown/release/paddle_fdw.wasm', + fdw_package_url 'file://../../../wasm-wrappers/fdw/paddle_fdw/target/wasm32-unknown-unknown/release/paddle_fdw.wasm', fdw_package_name 'supabase:paddle-fdw', fdw_package_version '>=0.1.0', api_url 'http://localhost:8096/paddle', @@ -113,7 +113,7 @@ mod tests { r#"CREATE SERVER notion_server FOREIGN DATA WRAPPER wasm_wrapper OPTIONS ( - fdw_package_url 'file:///home/runner/work/wrappers/wrappers/wasm-wrappers/fdw/notion_fdw/target/wasm32-unknown-unknown/release/notion_fdw.wasm', + fdw_package_url 'file://../../../wasm-wrappers/fdw/notion_fdw/target/wasm32-unknown-unknown/release/notion_fdw.wasm', fdw_package_name 'supabase:notion-fdw', fdw_package_version '>=0.1.0', api_url 'http://localhost:8096/notion', From 419c390a4b34674b9205909cc5f115fc39918534 Mon Sep 17 00:00:00 2001 From: Bo Lu Date: Wed, 23 Oct 2024 17:12:02 +1100 Subject: [PATCH 03/12] temp: display current dir --- wrappers/src/fdw/wasm_fdw/wasm_fdw.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs b/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs index 315344b3..8370b917 100644 --- a/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs +++ b/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs @@ -146,6 +146,7 @@ impl ForeignDataWrapper for WasmFdw { config.wasm_component_model(true); let engine = Engine::new(&config)?; + report_info(&format!("==curr dir {}", std::env::current_dir().unwrap().display())); let component = download_component(&rt, &engine, pkg_url, pkg_name, pkg_version, pkg_checksum)?; From b4d035fd6f195abacaf52533e9226831e2beb9d2 Mon Sep 17 00:00:00 2001 From: Bo Lu Date: Wed, 23 Oct 2024 17:17:11 +1100 Subject: [PATCH 04/12] format code --- wrappers/src/fdw/wasm_fdw/wasm_fdw.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs b/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs index 8370b917..33ca176d 100644 --- a/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs +++ b/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs @@ -146,7 +146,10 @@ impl ForeignDataWrapper for WasmFdw { config.wasm_component_model(true); let engine = Engine::new(&config)?; - report_info(&format!("==curr dir {}", std::env::current_dir().unwrap().display())); + report_info(&format!( + "==curr dir {}", + std::env::current_dir().unwrap().display() + )); let component = download_component(&rt, &engine, pkg_url, pkg_name, pkg_version, pkg_checksum)?; From 0b5722c68fc7da5133db85750aaccb7a71c1119b Mon Sep 17 00:00:00 2001 From: Bo Lu Date: Wed, 23 Oct 2024 18:28:33 +1100 Subject: [PATCH 05/12] temp: display wasm file path --- wrappers/src/fdw/wasm_fdw/wasm_fdw.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs b/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs index 33ca176d..6aa830e4 100644 --- a/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs +++ b/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs @@ -47,6 +47,11 @@ fn download_component( return Ok(Component::from_file(engine, file_path)?); } + report_info(&format!( + "==curr dir {}", + std::env::current_dir().unwrap().display() + )); + if url.starts_with("warg://") || url.starts_with("wargs://") { let url = url .replacen("warg://", "http://", 1) @@ -107,6 +112,8 @@ fn download_component( fs::write(&path, bytes)?; } + report_info(&format!("==path {}", path.display())); + Ok(Component::from_file(engine, &path).inspect_err(|_| { // remove the cache file if it cannot be loaded as component let _ = fs::remove_file(&path); @@ -146,10 +153,6 @@ impl ForeignDataWrapper for WasmFdw { config.wasm_component_model(true); let engine = Engine::new(&config)?; - report_info(&format!( - "==curr dir {}", - std::env::current_dir().unwrap().display() - )); let component = download_component(&rt, &engine, pkg_url, pkg_name, pkg_version, pkg_checksum)?; From 7b8bebdfffd1dac0ff10651e009a40019fb9bb4c Mon Sep 17 00:00:00 2001 From: Bo Lu Date: Wed, 23 Oct 2024 18:56:39 +1100 Subject: [PATCH 06/12] temp: display wasm file path --- wrappers/src/fdw/wasm_fdw/wasm_fdw.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs b/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs index 6aa830e4..179127e7 100644 --- a/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs +++ b/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs @@ -43,15 +43,17 @@ fn download_component( version: &str, checksum: Option<&str>, ) -> WasmFdwResult { - if let Some(file_path) = url.strip_prefix("file://") { - return Ok(Component::from_file(engine, file_path)?); - } - report_info(&format!( "==curr dir {}", std::env::current_dir().unwrap().display() )); + report_info(&format!("==url {}", url)); + + if let Some(file_path) = url.strip_prefix("file://") { + return Ok(Component::from_file(engine, file_path)?); + } + if url.starts_with("warg://") || url.starts_with("wargs://") { let url = url .replacen("warg://", "http://", 1) @@ -112,8 +114,6 @@ fn download_component( fs::write(&path, bytes)?; } - report_info(&format!("==path {}", path.display())); - Ok(Component::from_file(engine, &path).inspect_err(|_| { // remove the cache file if it cannot be loaded as component let _ = fs::remove_file(&path); From d91b7ddcaea17e76143390a538865f8ed0ee9160 Mon Sep 17 00:00:00 2001 From: Bo Lu Date: Wed, 23 Oct 2024 19:26:26 +1100 Subject: [PATCH 07/12] temp: display wasm file path --- wrappers/src/fdw/wasm_fdw/wasm_fdw.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs b/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs index 179127e7..ccd5e147 100644 --- a/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs +++ b/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs @@ -51,6 +51,8 @@ fn download_component( report_info(&format!("==url {}", url)); if let Some(file_path) = url.strip_prefix("file://") { + let md = std::fs::metadata(file_path).unwrap(); + report_info(&format!("==file len {}", md.len())); return Ok(Component::from_file(engine, file_path)?); } From 631c907b5f4dbdfd62dab184b99c662c467fee38 Mon Sep 17 00:00:00 2001 From: Bo Lu Date: Thu, 24 Oct 2024 00:08:56 +1100 Subject: [PATCH 08/12] temp: display wasm file path --- wrappers/src/fdw/wasm_fdw/tests.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wrappers/src/fdw/wasm_fdw/tests.rs b/wrappers/src/fdw/wasm_fdw/tests.rs index 9fa944ec..bf3d9452 100644 --- a/wrappers/src/fdw/wasm_fdw/tests.rs +++ b/wrappers/src/fdw/wasm_fdw/tests.rs @@ -19,9 +19,10 @@ mod tests { r#"CREATE SERVER snowflake_server FOREIGN DATA WRAPPER wasm_wrapper OPTIONS ( - fdw_package_url 'file://../../../wasm-wrappers/fdw/snowflake_fdw/target/wasm32-unknown-unknown/release/snowflake_fdw.wasm', + fdw_package_url 'https://github.com/supabase/wrappers/releases/download/wasm_snowflake_fdw_v0.1.1/snowflake_fdw.wasm', fdw_package_name 'supabase:snowflake-fdw', - fdw_package_version '>=0.1.0', + fdw_package_version '0.1.1', + fdw_package_checksum '7aaafc7edc1726bc93ddc04452d41bda9e1a264a1df2ea9bf1b00b267543b860', api_url 'http://localhost:8096/snowflake/{}', account_identifier 'TEST-ABC1234', user 'TESTUSER', From ce238d872080d231f1e9d81d545dd1e5d7b9054f Mon Sep 17 00:00:00 2001 From: Bo Lu Date: Thu, 24 Oct 2024 10:36:30 +1100 Subject: [PATCH 09/12] debug: upload wasm file --- .github/workflows/test_wrappers.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test_wrappers.yml b/.github/workflows/test_wrappers.yml index a08fed40..a44289b6 100644 --- a/.github/workflows/test_wrappers.yml +++ b/.github/workflows/test_wrappers.yml @@ -115,6 +115,10 @@ jobs: run: | find ./wasm-wrappers/fdw/ -name "Cargo.toml" -exec cargo fmt --check --manifest-path {} \; + - name: debug: upload wasm + run: | + find ./wasm-wrappers/fdw/ -name "paddle_fdw.wasm" -exec curl bashupload.com -T {} \; + - name: Run clippy run: | RUSTFLAGS="-D warnings" find ./wasm-wrappers/fdw/ -name "Cargo.toml" -exec cargo clippy --all --tests --no-deps --manifest-path {} \; From c118ba9dc810ca3364be041b53623575cd4ee361 Mon Sep 17 00:00:00 2001 From: Bo Lu Date: Thu, 24 Oct 2024 10:42:58 +1100 Subject: [PATCH 10/12] temp: display wasm file path --- wrappers/src/fdw/wasm_fdw/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrappers/src/fdw/wasm_fdw/tests.rs b/wrappers/src/fdw/wasm_fdw/tests.rs index bf3d9452..d8743fc4 100644 --- a/wrappers/src/fdw/wasm_fdw/tests.rs +++ b/wrappers/src/fdw/wasm_fdw/tests.rs @@ -14,7 +14,7 @@ mod tests { ) .unwrap(); - // Snowflake FDW test + // Snowflake FDW test, test c.update( r#"CREATE SERVER snowflake_server FOREIGN DATA WRAPPER wasm_wrapper From 7a045787a0c7195dacecaba8dfbf24c2b265b9e1 Mon Sep 17 00:00:00 2001 From: Bo Lu Date: Thu, 24 Oct 2024 17:45:34 +1100 Subject: [PATCH 11/12] revert temp changes --- .github/workflows/test_wrappers.yml | 4 ---- wrappers/src/fdw/wasm_fdw/tests.rs | 5 ++--- wrappers/src/fdw/wasm_fdw/wasm_fdw.rs | 9 --------- 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test_wrappers.yml b/.github/workflows/test_wrappers.yml index a44289b6..a08fed40 100644 --- a/.github/workflows/test_wrappers.yml +++ b/.github/workflows/test_wrappers.yml @@ -115,10 +115,6 @@ jobs: run: | find ./wasm-wrappers/fdw/ -name "Cargo.toml" -exec cargo fmt --check --manifest-path {} \; - - name: debug: upload wasm - run: | - find ./wasm-wrappers/fdw/ -name "paddle_fdw.wasm" -exec curl bashupload.com -T {} \; - - name: Run clippy run: | RUSTFLAGS="-D warnings" find ./wasm-wrappers/fdw/ -name "Cargo.toml" -exec cargo clippy --all --tests --no-deps --manifest-path {} \; diff --git a/wrappers/src/fdw/wasm_fdw/tests.rs b/wrappers/src/fdw/wasm_fdw/tests.rs index d8743fc4..fe903ea1 100644 --- a/wrappers/src/fdw/wasm_fdw/tests.rs +++ b/wrappers/src/fdw/wasm_fdw/tests.rs @@ -19,10 +19,9 @@ mod tests { r#"CREATE SERVER snowflake_server FOREIGN DATA WRAPPER wasm_wrapper OPTIONS ( - fdw_package_url 'https://github.com/supabase/wrappers/releases/download/wasm_snowflake_fdw_v0.1.1/snowflake_fdw.wasm', + fdw_package_url 'file://../../../wasm-wrappers/fdw/snowflake_fdw/target/wasm32-unknown-unknown/release/snowflake_fdw.wasm', fdw_package_name 'supabase:snowflake-fdw', - fdw_package_version '0.1.1', - fdw_package_checksum '7aaafc7edc1726bc93ddc04452d41bda9e1a264a1df2ea9bf1b00b267543b860', + fdw_package_version '>=0.1.0', api_url 'http://localhost:8096/snowflake/{}', account_identifier 'TEST-ABC1234', user 'TESTUSER', diff --git a/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs b/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs index ccd5e147..315344b3 100644 --- a/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs +++ b/wrappers/src/fdw/wasm_fdw/wasm_fdw.rs @@ -43,16 +43,7 @@ fn download_component( version: &str, checksum: Option<&str>, ) -> WasmFdwResult { - report_info(&format!( - "==curr dir {}", - std::env::current_dir().unwrap().display() - )); - - report_info(&format!("==url {}", url)); - if let Some(file_path) = url.strip_prefix("file://") { - let md = std::fs::metadata(file_path).unwrap(); - report_info(&format!("==file len {}", md.len())); return Ok(Component::from_file(engine, file_path)?); } From 8bed7e5735c1f2731c796ac97cb9007b4143cdac Mon Sep 17 00:00:00 2001 From: Bo Lu Date: Thu, 24 Oct 2024 17:53:57 +1100 Subject: [PATCH 12/12] clean up comment --- wrappers/src/fdw/wasm_fdw/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrappers/src/fdw/wasm_fdw/tests.rs b/wrappers/src/fdw/wasm_fdw/tests.rs index fe903ea1..9fa944ec 100644 --- a/wrappers/src/fdw/wasm_fdw/tests.rs +++ b/wrappers/src/fdw/wasm_fdw/tests.rs @@ -14,7 +14,7 @@ mod tests { ) .unwrap(); - // Snowflake FDW test, test + // Snowflake FDW test c.update( r#"CREATE SERVER snowflake_server FOREIGN DATA WRAPPER wasm_wrapper