diff --git a/Cargo.toml b/Cargo.toml index 39eacb61..1990f97c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ exclude = [ "wasm-wrappers/fdw/helloworld_fdw", "wasm-wrappers/fdw/snowflake_fdw", "wasm-wrappers/fdw/paddle_fdw", + "wasm-wrappers/fdw/notion_fdw", ] resolver = "2" diff --git a/README.md b/README.md index 82c71e5a..612aa6a4 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ | [SQL Server](./wrappers/src/fdw/mssql_fdw) | A FDW for [Microsoft SQL Server](https://www.microsoft.com/en-au/sql-server/) | ✅ | ❌ | | [Redis](./wrappers/src/fdw/redis_fdw) | A FDW for [Redis](https://redis.io/) | ✅ | ❌ | | [AWS Cognito](./wrappers/src/fdw/cognito_fdw) | A FDW for [AWS Cognito](https://aws.amazon.com/cognito/) | ✅ | ❌ | -| [Notion](./wrappers/src/fdw/notion_fdw) | A FDW for [Notion](https://www.notion.so/) | ✅ | ❌ | +| [Notion](./wasm-wrappers/fdw/notion_fdw) | A Wasm FDW for [Notion](https://www.notion.so/) | ✅ | ❌ | | [Snowflake](./wasm-wrappers/fdw/snowflake_fdw) | A Wasm FDW for [Snowflake](https://www.snowflake.com/) | ✅ | ✅ | | [Paddle](./wasm-wrappers/fdw/paddle_fdw) | A Wasm FDW for [Paddle](https://www.paddle.com/) | ✅ | ✅ | diff --git a/docs/catalog/index.md b/docs/catalog/index.md index 4b2c2c4b..5d44ddbe 100644 --- a/docs/catalog/index.md +++ b/docs/catalog/index.md @@ -34,3 +34,4 @@ See [Developing a Wasm Wrapper](../guides/create-wasm-wrapper.md) for instructio | ----------- | :------------------------------: | :----------------------------------: | :------------------------------------------------------------------------------------: | | Paddle | [Supabase](https://supabase.com) | [Link](paddle.md) | [Link](https://github.com/supabase/wrappers/tree/main/wasm-wrappers/fdw/paddle_fdw) | | Snowflake | [Supabase](https://supabase.com) | [Link](snowflake.md) | [Link](https://github.com/supabase/wrappers/tree/main/wasm-wrappers/fdw/snowflake_fdw) | +| Notion | [Supabase](https://supabase.com) | [Link](notion.md) | [Link](https://github.com/supabase/wrappers/tree/main/wasm-wrappers/fdw/notion_fdw) | diff --git a/docs/catalog/notion.md b/docs/catalog/notion.md index 8804aca9..4b020aa6 100644 --- a/docs/catalog/notion.md +++ b/docs/catalog/notion.md @@ -4,144 +4,302 @@ documentation: author: supabase tags: - native - - private alpha + - official --- # Notion [Notion](https://notion.so/) provides a versatile, ready-to-use solution for managing your data. -The Notion Wrapper allows you to read data from your Notion workspace for use within your Postgres database. Only the users endpoint is supported at the moment. +The Notion Wrapper is a WebAssembly(Wasm) foreign data wrapper which allows you to read data from your Notion workspace for use within your Postgres database. !!! warning Restoring a logical backup of a database with a materialized view using a foreign table can fail. For this reason, either do not use foreign tables in materialized views or use them in databases with physical backups enabled. +## Supported Data Types + +| Postgres Data Type | Notion Data Type | +| ------------------ | ---------------- | +| boolean | Boolean | +| text | String | +| timestamp | Time | +| timestamptz | Time | +| jsonb | Json | + +The Notion API uses JSON formatted data, please refer to [Notion API docs](https://developers.notion.com/reference/intro) for more details. + +## Available Versions + +| Version | Wasm Package URL | Checksum | +| ------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------------------------ | +| 0.1.0 | `https://github.com/supabase/wrappers/releases/download/wasm_notion_fdw_v0.1.0/notion_fdw.wasm` | `tbd` | + ## Preparation Before you get started, make sure the `wrappers` extension is installed on your database: ```sql -CREATE extension IF NOT EXISTS wrappers SCHEMA extensions; +create extension if not exists wrappers with schema extensions; ``` -Then, create the foreign data wrapper: +and then create the Wasm foreign data wrapper: ```sql -CREATE FOREIGN DATA WRAPPER notion_wrapper - HANDLER notion_fdw_handler - VALIDATOR notion_fdw_validator; +create foreign data wrapper wasm_wrapper + handler wasm_fdw_handler + validator wasm_fdw_validator; ``` ### Secure your credentials (optional) -By default, Postgres stores FDW credentials in plain text within the `pg_catalog.pg_foreign_server` table, making them visible to anyone with access to this table. To enhance security, it is advisable to use [Vault](https://supabase.com/docs/guides/database/vault) for credential storage. Vault integrates seamlessly with Wrappers to provide a secure storage solution for sensitive information. We strongly recommend utilizing Vault to safeguard your credentials. +By default, Postgres stores FDW credentials inside `pg_catalog.pg_foreign_server` in plain text. Anyone with access to this table will be able to view these credentials. Wrappers is designed to work with [Vault](https://supabase.com/docs/guides/database/vault), which provides an additional level of security for storing credentials. We recommend using Vault to store your credentials. ```sql -- Save your Notion API key in Vault and retrieve the `key_id` -INSERT INTO vault.secrets (name, secret) -VALUES ( +insert into vault.secrets (name, secret) +values ( 'notion', '' -- Notion API key ) -RETURNING key_id; +returning key_id; ``` ### Connecting to Notion -We need to provide Postgres with the credentials to connect to Notion, and any additional options. We can do this using the `CREATE SERVER` command: - -- With Vault (recommended) - - ```sql - CREATE SERVER notion_server - FOREIGN DATA WRAPPER notion_wrapper - OPTIONS ( - api_key_id '', -- The Key ID from the Vault - notion_version '', -- optional, default is '2022-06-28' - api_url '' -- optional, default is 'https://api.notion.com/v1/' - ); - ``` - -- Without Vault - - ```sql - CREATE SERVER notion_server - FOREIGN DATA WRAPPER notion_wrapper - OPTIONS ( - api_key '', -- Your Notion API key - notion_version '', -- optional, default is '2022-06-28' - api_url '' -- optional, default is 'https://api.notion.com/v1/' - ); - ``` +We need to provide Postgres with the credentials to access Notion, and any additional options. We can do this using the `create server` command: + +=== "With Vault" + + ```sql + create server notion_server + foreign data wrapper wasm_wrapper + options ( + fdw_package_url 'https://github.com/supabase/wrappers/releases/download/wasm_notion_fdw_v0.1.0/notion_fdw.wasm', + fdw_package_name 'supabase:notion-fdw', + fdw_package_version '0.1.0', + fdw_package_checksum 'tbd', + api_url 'https://api.notion.com/v1', -- optional + api_version '2022-06-28', -- optional + api_key_id '' -- The Key ID from above. + ); + ``` + +=== "Without Vault" + + ```sql + create server notion_server + foreign data wrapper wasm_wrapper + options ( + fdw_package_url 'https://github.com/supabase/wrappers/releases/download/wasm_notion_fdw_v0.1.0/notion_fdw.wasm', + fdw_package_name 'supabase:notion-fdw', + fdw_package_version '0.1.0', + fdw_package_checksum 'tbd', + api_url 'https://api.notion.com/v1', -- optional + api_version '2022-06-28', -- optional + api_key 'secret_xxxxx' -- Notion API key + ); + ``` + +Note the `fdw_package_*` options are required, which specify the Wasm package metadata. You can get the available package version list from [above](#available-versions). + +### Create a schema + +We recommend creating a schema to hold all the foreign tables: + +```sql +create schema if not exists notion; +``` ## Creating Foreign Tables -The Notion Wrapper supports data reads from the [Notion API](https://developers.notion.com/reference). +The Notion Wrapper supports data reads from below objects in Notion. -| Object | Select | Insert | Update | Delete | Truncate | -| ---------------------------------------------------------- | :----: | :----: | :----: | :----: | :------: | -| [Users](https://developers.notion.com/reference/get-users) | ✅ | ❌ | ❌ | ❌ | ❌ | +| Integration | Select | Insert | Update | Delete | Truncate | +| ----------- | :----: | :----: | :----: | :----: | :------: | +| Block | ✅ | ❌ | ❌ | ❌ | ❌ | +| Page | ✅ | ❌ | ❌ | ❌ | ❌ | +| Database | ✅ | ❌ | ❌ | ❌ | ❌ | +| User | ✅ | ❌ | ❌ | ❌ | ❌ | For example: ```sql -CREATE FOREIGN TABLE my_foreign_table ( +-- Note: the 'page_id' isn't presented in the Notion Block's fields, it is +-- added by the foreign data wrapper for development convenience. All blocks, +-- including nested children blocks, belong to one page will have a same +-- 'page_id' of that page. +create foreign table notion.blocks ( + id text, + page_id text, + type text, + created_time timestamp, + last_edited_time timestamp, + archived boolean, + attrs jsonb +) + server notion_server + options ( + object 'block' + ); + +create foreign table notion.pages ( + id text, + url text, + created_time timestamp, + last_edited_time timestamp, + archived boolean, + attrs jsonb +) + server notion_server + options ( + object 'page' + ); + +create foreign table notion.databases ( + id text, + url text, + created_time timestamp, + last_edited_time timestamp, + archived boolean, + attrs jsonb +) + server notion_server + options ( + object 'database' + ); + +create foreign table notion.users ( id text, name text, type text, - person jsonb, - bot jsonb - -- other fields + avatar_url text, + attrs jsonb ) - SERVER notion_server - OPTIONS ( - object 'users', -); + server notion_server + options ( + object 'user' + ); ``` +!!! note + + - All the supported columns are listed above, other columns are not allowd. + - The `attrs` is a special column which stores all the object attributes in JSON format, you can extract any attributes needed from it. See more examples below. + +### Foreign table options + +The full list of foreign table options are below: + +- `object` - Object name in Notion, required. + + Supported objects are listed below: + + | Object name | + | --------------------- | + | block | + | page | + | database | + | user | + ## Query Pushdown Support -This FDW supports `where` clause pushdown. You can specify a filter in `where` clause and it will be passed to Notion API call. +This FDW supports `where` clause pushdown with `id` as the filter. For example, + +```sql +select * from notion.pages +where id = '5a67c86f-d0da-4d0a-9dd7-f4cf164e6247'; +``` + +will be translated Notion API call: `https://api.notion.com/v1/pages/5a67c86f-d0da-4d0a-9dd7-f4cf164e6247`. -For example, this query +In addition to `id` column pushdown, `page_id` column pushdown is also supported for `Block` object. For example, ```sql -SELECT * from notion_users where id = 'xxx'; +select * from notion.blocks +where page_id = '5a67c86f-d0da-4d0a-9dd7-f4cf164e6247'; ``` -will be translated Notion API call: `https://api.notion.com/v1/users/xxx`. +will recursively fetch all children blocks of the Page with id '5a67c86f-d0da-4d0a-9dd7-f4cf164e6247'. This can dramatically reduce number of API calls and improve query speed. + +!!! note + + Below query will request ALL the blocks of ALL pages recursively, it may take very long time to run if there are many pages in Notion. So it is recommended to always query Block object with an `id` or `page_id` filter like above. + + ```sql + select * from notion.blocks; + ``` ## Examples -Some examples on how to use Notion foreign tables. +Below are some examples on how to use Notion foreign tables. + +### Basic example -### Users foreign table +This example will create a "foreign table" inside your Postgres database and query its data. First, we can create a schema to hold all the Notion foreign tables. + +```sql +create schema if not exists notion; +``` -The following command creates a "foreign table" in your Postgres database named `notion_users`: +Then create the foreign table and query it, for example: ```sql -CREATE FOREIGN TABLE notion_users ( +create foreign table notion.pages ( id text, - name text, - type text, - person jsonb, - bot jsonb + url text, + created_time timestamp, + last_edited_time timestamp, + archived boolean, + attrs jsonb ) - SERVER notion_server - OPTIONS ( - object 'users' + server notion_server + options ( + object 'page' ); + +-- query all pages +select * from notion.pages; + +-- query one page +select * from notion.pages +where id = '5a67c86f-d0da-4d0a-9dd7-f4cf164e6247'; ``` -You can now fetch your Notion data from within your Postgres database: +`attrs` is a special column which stores all the object attributes in JSON format, you can extract any attributes needed from it. See more examples below. + +### Query JSON attributes ```sql -SELECT * FROM notion_users; +create foreign table notion.users ( + id text, + name text, + type text, + avatar_url text, + attrs jsonb +) + server notion_server + options ( + object 'user' + ); + +-- extract user's email address +select id, attrs->'person'->>'email' as email +from notion.users +where id = 'fd0ed76c-44bd-413a-9448-18ff4b1d6a5e'; ``` -You can also query with filters: +### Query Blocks ```sql -SELECT * FROM notion_users WHERE id = 'xxx'; +-- query ALL blocks of ALL pages recursively, may take long time! +select * from notion.blocks; + +-- query a single block by block id +select * from notion.blocks +where id = 'fc248547-83ef-4069-b7c9-18897edb7150'; + +-- query all block of a page by page id +select * from notion.blocks +where page_id = '5a67c86f-d0da-4d0a-9dd7-f4cf164e6247'; ``` diff --git a/docs/catalog/paddle.md b/docs/catalog/paddle.md index 218f9adc..93aa8365 100644 --- a/docs/catalog/paddle.md +++ b/docs/catalog/paddle.md @@ -175,7 +175,7 @@ select * from paddle.customers where id = 'ctm_01hymwgpkx639a6mkvg99563sp'; ## Examples -Below are Some examples on how to use Paddle foreign tables. +Below are some examples on how to use Paddle foreign tables. ### Basic example diff --git a/docs/catalog/wasm/index.md b/docs/catalog/wasm/index.md index ba429e44..01c91ff3 100644 --- a/docs/catalog/wasm/index.md +++ b/docs/catalog/wasm/index.md @@ -37,4 +37,16 @@ Foreign data wrappers built with Wasm which can be used on Supabase platform. :octicons-code-24: [source](https://github.com/supabase/wrappers/tree/wasm_snowflake_fdw_v0.1.1/wasm-wrappers/fdw/snowflake_fdw)   :material-file-document: [docs](../snowflake.md) +- :simple-webassembly:   **[Notion](../notion.md)** + + ---- + + Foreign data wrapper for [Notion](https://notion.so/). + + Supported by [Supabase](https://www.supabase.com) + + :octicons-tag-24: [v0.1.0](https://github.com/supabase/wrappers/releases/tag/wasm_notion_fdw_v0.1.0)   + :octicons-code-24: [source](https://github.com/supabase/wrappers/tree/wasm_notion_fdw_v0.1.0/wasm-wrappers/fdw/notion_fdw)   + :material-file-document: [docs](../notion.md) + diff --git a/mkdocs.yaml b/mkdocs.yaml index e25ac9f1..386b9340 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -22,7 +22,6 @@ nav: - ClickHouse: 'catalog/clickhouse.md' - Firebase: 'catalog/firebase.md' - Logflare: 'catalog/logflare.md' - - Notion: 'catalog/notion.md' - Redis: 'catalog/redis.md' - S3 (CSV, JSON, Parquet): 'catalog/s3.md' - Stripe: 'catalog/stripe.md' @@ -31,6 +30,7 @@ nav: - catalog/wasm/index.md - Paddle: 'catalog/paddle.md' - Snowflake: 'catalog/snowflake.md' + - Notion: 'catalog/notion.md' - Guides: - Native vs Wasm Wrappers: 'guides/native-wasm.md' - Query Pushdown: 'guides/query-pushdown.md' diff --git a/wasm-wrappers/fdw/notion_fdw/Cargo.lock b/wasm-wrappers/fdw/notion_fdw/Cargo.lock new file mode 100644 index 00000000..6e33f0d6 --- /dev/null +++ b/wasm-wrappers/fdw/notion_fdw/Cargo.lock @@ -0,0 +1,358 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "autocfg" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "cc" +version = "1.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "js-sys" +version = "0.3.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.158" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "notion_fdw" +version = "0.1.0" +dependencies = [ + "chrono", + "serde_json", + "wit-bindgen-rt", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "serde" +version = "1.0.210" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.210" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" + +[[package]] +name = "wasm-bindgen" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen-rt" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c7526379ace8709ee9ab9f2bb50f112d95581063a59ef3097d9c10153886c9" diff --git a/wasm-wrappers/fdw/notion_fdw/Cargo.toml b/wasm-wrappers/fdw/notion_fdw/Cargo.toml new file mode 100644 index 00000000..9cb88965 --- /dev/null +++ b/wasm-wrappers/fdw/notion_fdw/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "notion_fdw" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +wit-bindgen-rt = "0.26.0" +serde_json = "1.0" +chrono = "0.4.38" + +[package.metadata.component] +package = "supabase:notion-fdw" + +[package.metadata.component.dependencies] + +[package.metadata.component.target] +path = "wit" + +[package.metadata.component.target.dependencies] +"supabase:wrappers" = { path = "../../wit" } diff --git a/wasm-wrappers/fdw/notion_fdw/src/lib.rs b/wasm-wrappers/fdw/notion_fdw/src/lib.rs new file mode 100644 index 00000000..15c52808 --- /dev/null +++ b/wasm-wrappers/fdw/notion_fdw/src/lib.rs @@ -0,0 +1,496 @@ +#[allow(warnings)] +mod bindings; +use serde_json::Value as JsonValue; + +use bindings::{ + exports::supabase::wrappers::routines::Guest, + supabase::wrappers::{ + http, stats, time, + types::{Cell, Column, Context, FdwError, FdwResult, OptionsType, Row, TypeOid, Value}, + utils, + }, +}; + +#[derive(Debug, Default)] +struct NotionFdw { + base_url: String, + headers: Vec<(String, String)>, + object: String, + src_rows: Vec, + src_idx: usize, +} + +static mut INSTANCE: *mut NotionFdw = std::ptr::null_mut::(); +static FDW_NAME: &str = "NotionFdw"; + +impl NotionFdw { + fn init() { + let instance = Self::default(); + unsafe { + INSTANCE = Box::leak(Box::new(instance)); + } + } + + fn this_mut() -> &'static mut Self { + unsafe { &mut (*INSTANCE) } + } + + // convert Notion response data field to a cell + // ref: https://developers.notion.com/reference/post-search + fn src_to_cell(&self, src_row: &JsonValue, tgt_col: &Column) -> Result, FdwError> { + let tgt_col_name = tgt_col.name(); + + // put all properties into 'attrs' JSON column + if &tgt_col_name == "attrs" { + return Ok(Some(Cell::Json(src_row.to_string()))); + } + + // only accept certain target column names, all the other properties will be put into + // 'attrs' JSON column + if !matches!( + tgt_col_name.as_str(), + "id" | "url" + | "created_time" + | "last_edited_time" + | "archived" + | "name" + | "type" + | "avatar_url" + | "page_id" + ) { + return Err(format!( + "target column name {} is not supported", + tgt_col_name + )); + } + + let src = src_row + .as_object() + .and_then(|v| v.get(&tgt_col_name)) + .ok_or(format!("source column '{}' not found", tgt_col_name))?; + + // column type mapping + let cell = match tgt_col.type_oid() { + TypeOid::Bool => src.as_bool().map(Cell::Bool), + TypeOid::String => src.as_str().map(|v| Cell::String(v.to_owned())), + TypeOid::Timestamp => { + if let Some(s) = src.as_str() { + let ts = time::parse_from_rfc3339(s)?; + Some(Cell::Timestamp(ts)) + } else { + None + } + } + TypeOid::Timestamptz => { + if let Some(s) = src.as_str() { + let ts = time::parse_from_rfc3339(s)?; + Some(Cell::Timestamptz(ts)) + } else { + None + } + } + TypeOid::Json => src.as_object().map(|_| Cell::Json(src.to_string())), + _ => { + return Err(format!( + "target column '{}' type is not supported", + tgt_col_name + )); + } + }; + + Ok(cell) + } + + // create a request instance + fn create_request( + &self, + object: &str, + block_id: Option<&str>, + start_cursor: &Option, + ctx: &Context, + ) -> Result { + let object_id = ctx + .get_quals() + .iter() + .find(|q| q.field() == "id") + .and_then(|id| { + // push down id filter + match id.value() { + Value::Cell(Cell::String(s)) => Some(s), + _ => None, + } + }); + + let (method, url, body) = match object { + // fetch databases or pages + "database" | "page" => { + if let Some(id) = object_id { + ( + http::Method::Get, + format!("{}/{}s/{}", self.base_url, object, id), + String::default(), + ) + } else { + let start_cursor_str = if let Some(ref sc) = start_cursor { + format!(r#""start_cursor": "{}","#, sc) + } else { + String::default() + }; + let body = format!( + r#"{{ + "query": "", + "filter": {{ + "value": "{}", + "property": "object" + }}, + "sort": {{ + "direction": "ascending", + "timestamp": "last_edited_time" + }}, + {} + "page_size": 100 + }}"#, + object, start_cursor_str, + ); + + ( + http::Method::Post, + format!("{}/search", self.base_url), + body, + ) + } + } + + // fetch users + "user" => { + if let Some(id) = object_id { + ( + http::Method::Get, + format!("{}/users/{}", self.base_url, id), + String::default(), + ) + } else { + let start_cursor = if let Some(ref sc) = start_cursor { + format!("&start_cursor={}", sc) + } else { + String::default() + }; + let url = format!("{}/users?page_size=100{}", self.base_url, start_cursor); + + (http::Method::Get, url, String::default()) + } + } + + // fetch blocks + "block" => { + if let Some(id) = object_id { + ( + http::Method::Get, + format!("{}/blocks/{}", self.base_url, id), + String::default(), + ) + } else if let Some(block_id) = block_id { + let start_cursor = if let Some(ref sc) = start_cursor { + format!("&start_cursor={}", sc) + } else { + String::default() + }; + let url = format!( + "{}/blocks/{}/children?page_size=100{}", + self.base_url, block_id, start_cursor + ); + (http::Method::Get, url, String::default()) + } else { + return Err("block id is not specified".to_owned()); + } + } + + _ => return Err(format!("object {} is not supported", object)), + }; + + Ok(http::Request { + method, + url, + headers: self.headers.clone(), + body, + }) + } + + // make request to Notion API, including following pagination requests + fn make_request( + &self, + object: &str, + block_id: Option<&str>, + ctx: &Context, + ) -> Result, FdwError> { + let mut ret: Vec = Vec::new(); + let mut start_cursor: Option = None; + + loop { + // create a request and send it + let req = self.create_request(object, block_id, &start_cursor, ctx)?; + let resp = match req.method { + http::Method::Get => http::get(&req)?, + http::Method::Post => http::post(&req)?, + _ => unreachable!("invalid request method"), + }; + + // idle for a while for retry when got rate limited error + // ref: https://developers.notion.com/reference/request-limits + if resp.status_code == 429 { + if let Some(retry) = resp.headers.iter().find(|h| h.0 == "retry-after") { + let delay = retry.1.parse::().map_err(|e| e.to_string())?; + time::sleep(delay * 1000); + continue; + } + } + + // transform response to json + let resp_json: JsonValue = + serde_json::from_str(&resp.body).map_err(|e| e.to_string())?; + + // if the 404 is caused by no object found, we shouldn't take it as an error + if resp.status_code == 404 + && resp_json.pointer("/code").and_then(|v| v.as_str()) == Some("object_not_found") + { + break; + } + + // check for errors + http::error_for_status(&resp).map_err(|err| format!("{}: {}", err, resp.body))?; + + // unify response object to array and save source rows + let resp_data = if resp_json.pointer("/object").and_then(|v| v.as_str()) == Some("list") + { + resp_json + .pointer("/results") + .and_then(|v| v.as_array().cloned()) + .ok_or("cannot get query result data")? + } else { + vec![resp_json.clone()] + }; + ret.extend(resp_data); + + stats::inc_stats(FDW_NAME, stats::Metric::BytesIn, resp.body.len() as i64); + + // deal with pagination to save next page cursor + start_cursor = resp_json + .pointer("/next_cursor") + .and_then(|v| v.as_str().map(|s| s.to_owned())); + if start_cursor.is_none() { + break; + } + + // Notion API rate limit is an average of three requests per second + // ref: https://developers.notion.com/reference/request-limits + time::sleep(350); + } + + Ok(ret) + } + + // recursively request a block and all of its children + fn request_blocks(&self, block_id: &str, ctx: &Context) -> Result, FdwError> { + let mut ret = Vec::new(); + + let children = self.make_request("block", Some(block_id), ctx)?; + + for child in children.iter() { + let has_children = child + .pointer("/has_children") + .and_then(|v| v.as_bool()) + .unwrap_or_default(); + if has_children { + if let Some(id) = child.pointer("/id").and_then(|v| v.as_str()) { + let child_blocks = self.request_blocks(id, ctx)?; + ret.extend(child_blocks); + } + } + } + + ret.extend(children); + + Ok(ret) + } + + // clone a block and add 'page_id' property to it for convenience + fn set_block_page_id(&self, block: &JsonValue, page_id: Option<&str>) -> JsonValue { + match block { + JsonValue::Object(b) => { + let page_id = if page_id.is_some() { + page_id + } else { + // if page_id is not specified, try to get it from block's 'parent' field + // ref: https://developers.notion.com/reference/retrieve-a-block + block.pointer("/parent/page_id").and_then(|v| v.as_str()) + }; + + let mut obj = b.clone(); + obj.insert( + "page_id".to_owned(), + if let Some(id) = page_id { + JsonValue::String(id.to_owned()) + } else { + JsonValue::Null + }, + ); + + JsonValue::Object(obj) + } + _ => JsonValue::Null, + } + } + + // fetch source data from Notion API + fn fetch_source_data(&mut self, ctx: &Context) -> FdwResult { + self.src_idx = 0; + + // for all non-block objects, request it straightly + if self.object != "block" { + self.src_rows = self.make_request(&self.object, None, ctx)?; + return Ok(()); + } + + // now for the block object, it can push down two fields: 'id' or 'page_id' + // so we need to deal with them separately + + let quals = ctx.get_quals(); + + if quals.iter().any(|q| q.field() == "id") { + // make request directly for 'id' qual push down + let blocks = self.make_request(&self.object, None, ctx)?; + self.src_rows = blocks + .iter() + .map(|b| self.set_block_page_id(b, None)) + .collect(); + } else if let Some(qual) = quals.iter().find(|q| q.field() == "page_id") { + // request all child blocks for a page if 'page_id' is pushed down + if let Value::Cell(Cell::String(page_id)) = qual.value() { + let blocks = self.request_blocks(page_id.as_str(), ctx)?; + self.src_rows = blocks + .iter() + .map(|b| self.set_block_page_id(b, Some(page_id.as_str()))) + .collect(); + } + } else { + // otherwise, we're querying all blocks, fetch all pages first + let pages = self.make_request("page", None, ctx)?; + + // fetch all the children blocks for each page + for page in pages.iter() { + if let Some(page_id) = page.pointer("/id").and_then(|v| v.as_str()) { + let blocks = self.request_blocks(page_id, ctx)?; + let blocks: Vec = blocks + .iter() + .map(|b| self.set_block_page_id(b, Some(page_id))) + .collect(); + self.src_rows.extend(blocks); + } + } + } + + Ok(()) + } +} + +impl Guest for NotionFdw { + fn host_version_requirement() -> String { + // semver ref: https://docs.rs/semver/latest/semver/enum.Op.html + "^0.1.0".to_string() + } + + fn init(ctx: &Context) -> FdwResult { + Self::init(); + let this = Self::this_mut(); + + // get foreign server options + let opts = ctx.get_options(OptionsType::Server); + this.base_url = opts.require_or("api_url", "https://api.notion.com/v1"); + let api_key = match opts.get("api_key") { + Some(key) => key, + None => { + let key_id = opts.require("api_key_id")?; + utils::get_vault_secret(&key_id).unwrap_or_default() + } + }; + let api_version = opts.require_or("api_version", "2022-06-28"); + + // Notion api authentication + // ref: https://developers.notion.com/docs/authorization + this.headers + .push(("user-agent".to_owned(), "Wrappers Notion FDW".to_string())); + this.headers + .push(("content-type".to_owned(), "application/json".to_string())); + this.headers + .push(("authorization".to_owned(), format!("Bearer {}", api_key))); + this.headers + .push(("notion-version".to_owned(), api_version)); + + stats::inc_stats(FDW_NAME, stats::Metric::CreateTimes, 1); + + Ok(()) + } + + fn begin_scan(ctx: &Context) -> FdwResult { + let this = Self::this_mut(); + let opts = ctx.get_options(OptionsType::Table); + this.object = opts.require("object")?; + + this.fetch_source_data(ctx) + } + + fn iter_scan(ctx: &Context, row: &Row) -> Result, FdwError> { + let this = Self::this_mut(); + + // if all source rows are consumed + if this.src_idx >= this.src_rows.len() { + stats::inc_stats(FDW_NAME, stats::Metric::RowsIn, this.src_rows.len() as i64); + stats::inc_stats(FDW_NAME, stats::Metric::RowsOut, this.src_rows.len() as i64); + return Ok(None); + } + + // convert Notion row to Postgres row + let src_row = &this.src_rows[this.src_idx]; + for tgt_col in ctx.get_columns() { + let cell = this.src_to_cell(src_row, &tgt_col)?; + row.push(cell.as_ref()); + } + + this.src_idx += 1; + + Ok(Some(0)) + } + + fn re_scan(ctx: &Context) -> FdwResult { + let this = Self::this_mut(); + this.fetch_source_data(ctx) + } + + fn end_scan(_ctx: &Context) -> FdwResult { + let this = Self::this_mut(); + this.src_rows.clear(); + Ok(()) + } + + fn begin_modify(_ctx: &Context) -> FdwResult { + Err("modify on foreign table is not supported".to_owned()) + } + + fn insert(_ctx: &Context, _row: &Row) -> FdwResult { + Ok(()) + } + + fn update(_ctx: &Context, _rowid: Cell, _row: &Row) -> FdwResult { + Ok(()) + } + + fn delete(_ctx: &Context, _rowid: Cell) -> FdwResult { + Ok(()) + } + + fn end_modify(_ctx: &Context) -> FdwResult { + Ok(()) + } +} + +bindings::export!(NotionFdw with_types_in bindings); diff --git a/wasm-wrappers/fdw/notion_fdw/wit/world.wit b/wasm-wrappers/fdw/notion_fdw/wit/world.wit new file mode 100644 index 00000000..3b81bdc2 --- /dev/null +++ b/wasm-wrappers/fdw/notion_fdw/wit/world.wit @@ -0,0 +1,10 @@ +package supabase:notion-fdw@0.1.0; + +world notion { + import supabase:wrappers/http@0.1.0; + import supabase:wrappers/jwt@0.1.0; + import supabase:wrappers/stats@0.1.0; + import supabase:wrappers/time@0.1.0; + import supabase:wrappers/utils@0.1.0; + export supabase:wrappers/routines@0.1.0; +} diff --git a/wasm-wrappers/fdw/paddle_fdw/Cargo.lock b/wasm-wrappers/fdw/paddle_fdw/Cargo.lock index 1a53d1c5..1e899da5 100644 --- a/wasm-wrappers/fdw/paddle_fdw/Cargo.lock +++ b/wasm-wrappers/fdw/paddle_fdw/Cargo.lock @@ -128,7 +128,7 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "paddle_fdw" -version = "0.1.0" +version = "0.1.1" dependencies = [ "chrono", "serde_json", diff --git a/wasm-wrappers/fdw/snowflake_fdw/Cargo.lock b/wasm-wrappers/fdw/snowflake_fdw/Cargo.lock index ae485752..88fea209 100644 --- a/wasm-wrappers/fdw/snowflake_fdw/Cargo.lock +++ b/wasm-wrappers/fdw/snowflake_fdw/Cargo.lock @@ -65,7 +65,7 @@ dependencies = [ [[package]] name = "snowflake_fdw" -version = "0.1.0" +version = "0.1.1" dependencies = [ "serde_json", "wit-bindgen-rt", diff --git a/wrappers/Cargo.toml b/wrappers/Cargo.toml index 0073bd9a..fec85b37 100644 --- a/wrappers/Cargo.toml +++ b/wrappers/Cargo.toml @@ -135,16 +135,6 @@ wasm_fdw = [ "serde_json", "jwt-simple", ] -notion_fdw = [ - "reqwest", - "reqwest-middleware", - "reqwest-retry", - "http", - "serde_json", - "serde", - "url", - "thiserror", -] # Does not include helloworld_fdw because of its general uselessness native_fdws = [ "airtable_fdw", diff --git a/wrappers/README.md b/wrappers/README.md index c03f7817..a897680b 100644 --- a/wrappers/README.md +++ b/wrappers/README.md @@ -14,4 +14,3 @@ This is a collection of FDWs built by [Supabase](https://www.supabase.com). We c - [Cognito](./src/fdw/cognito_fdw): A FDW for [AWS Cogntio](https://aws.amazon.com/pm/cognito/). - [SQL Server](./src/fdw/mssql_fdw): A FDW for [Microsoft SQL Server](https://www.microsoft.com/en-au/sql-server/) which supports data read only. - [Redis](./src/fdw/redis_fdw): A FDW for [Redis](https://redis.io/) which supports data read only. -- [Notion](./src/fdw/notion_fdw): A FDW for [Notion](https://notion.so/) which supports users read only. diff --git a/wrappers/dockerfiles/wasm/server.py b/wrappers/dockerfiles/wasm/server.py index 9b9ce2e2..9b1a831e 100644 --- a/wrappers/dockerfiles/wasm/server.py +++ b/wrappers/dockerfiles/wasm/server.py @@ -65,6 +65,59 @@ def do_GET(self): "estimated_total": 2 } } +} + ''' + elif fdw == "notion": + body = ''' +{ + "object": "page", + "id": "5a67c86f-d0da-4d0a-9dd7-f4cf164e6247", + "created_time": "2021-10-15T05:41:00.000Z", + "last_edited_time": "2021-10-15T05:49:00.000Z", + "created_by": { + "object": "user", + "id": "fd0ed76c-44bd-413a-9448-18ff4b1d6a5e" + }, + "last_edited_by": { + "object": "user", + "id": "fd0ed76c-44bd-413a-9448-18ff4b1d6a5e" + }, + "cover": null, + "icon": null, + "parent": { + "type": "workspace", + "workspace": true + }, + "archived": false, + "in_trash": false, + "properties": { + "title": { + "id": "title", + "type": "title", + "title": [ + { + "type": "text", + "text": { + "content": "test page3", + "link": null + }, + "annotations": { + "bold": false, + "italic": false, + "strikethrough": false, + "underline": false, + "code": false, + "color": "default" + }, + "plain_text": "test page3", + "href": null + } + ] + } + }, + "url": "https://www.notion.so/test-page3-5a67c86fd0da4d0a9dd7f4cf164e6247", + "public_url": null, + "request_id": "85a75f82-bd22-414e-a3a7-5c00a9451a1c" } ''' else: diff --git a/wrappers/src/fdw/mod.rs b/wrappers/src/fdw/mod.rs index 9d317747..d10de110 100644 --- a/wrappers/src/fdw/mod.rs +++ b/wrappers/src/fdw/mod.rs @@ -36,6 +36,3 @@ mod cognito_fdw; #[cfg(feature = "wasm_fdw")] mod wasm_fdw; - -#[cfg(feature = "notion_fdw")] -mod notion_fdw; diff --git a/wrappers/src/fdw/notion_fdw/README.md b/wrappers/src/fdw/notion_fdw/README.md deleted file mode 100644 index 7bebe7dd..00000000 --- a/wrappers/src/fdw/notion_fdw/README.md +++ /dev/null @@ -1,88 +0,0 @@ -# Notion Foreign Data Wrapper - -This is a foreign data wrapper for [Notion](https://notion.so/) developed using [Wrappers](https://github.com/supabase/wrappers). - -## Documentation - -[https://fdw.dev/catalog/notion](https://fdw.dev/catalog/notion/) - -## Basic usage - -These steps outline how to use the this FDW: - -1. Clone this repo - -```bash -git clone https://github.com/supabase/wrappers.git -``` - -2. Run it using pgrx with feature: - -```bash -cd wrappers/wrappers -cargo pgrx run --features notion_fdw -``` - -3. Create the extension, foreign data wrapper and related objects: - -```sql --- Create the extension (if not already created) -CREATE extension wrappers; -``` - -```sql --- Create the foreign data wrapper and specify the handler and validator functions -CREATE FOREIGN DATA WRAPPER notion_wrapper - HANDLER notion_fdw_handler - VALIDATOR notion_fdw_validator; -``` - -```sql --- Create a server object with the necessary options -CREATE SERVER notion_server - FOREIGN DATA WRAPPER notion_wrapper - OPTIONS ( - api_key '', - notion_version '2022-06-28', -- optional, default is '2022-06-28' - api_url 'https://api.notion.com/v1/' -- optional, default is 'https://api.notion.com/v1/' - ); -``` - -```sql --- Create an example foreign tabl --- The number of fields are illustrative -CREATE FOREIGN TABLE notion_users ( - id text, - name text, - type text, - person jsonb, - bot jsonb -) SERVER notion_server OPTIONS ( - object 'users' -); -``` - - -4. Run a query to check if it is working: - -```bash -wrappers=# SELECT * FROM notion_users; --[ RECORD 1 ]-------------------------------------------------------------------------------------------- -id | ad4d1b3b-4f3b-4b7e-8f1b-3f1f2f3f1f2f -name | John Doe -type | person -person | {"email": "john@doe.com", "name": "John Doe"} -bot | --[ RECORD 2 ]-------------------------------------------------------------------------------------------- -id | 45f3b4b3-4f3b-4b7e-8f1b-12a3b4c5d6e7 -name | Beep Boop -type | bot -person | -bot | {"owner": {"type": "workspace", "workspace": true}, "workspace_name": "John's workspace"} -```` - -## Changelog - -| Version | Date | Notes | -| ------- | ---------- | --------------------------------------------- | -| 0.1.0 | 2024-05-13 | Initial version with users capabilities only. | diff --git a/wrappers/src/fdw/notion_fdw/mod.rs b/wrappers/src/fdw/notion_fdw/mod.rs deleted file mode 100644 index c1576cdf..00000000 --- a/wrappers/src/fdw/notion_fdw/mod.rs +++ /dev/null @@ -1,51 +0,0 @@ -#![allow(clippy::module_inception)] -mod notion_fdw; -mod tests; - -use http::header::InvalidHeaderValue; -use pgrx::pg_sys::panic::ErrorReport; -use pgrx::prelude::PgSqlErrorCode; -use thiserror::Error; - -use supabase_wrappers::prelude::{CreateRuntimeError, OptionsError}; - -#[derive(Error, Debug)] -enum NotionFdwError { - #[error("Notion object '{0}' not yet implemented")] - ObjectNotImplemented(String), - - #[error("invalid header: {0}")] - InvalidApiKeyHeader(#[from] InvalidHeaderValue), - - #[error("request failed: {0}")] - RequestError(#[from] reqwest::Error), - - #[error("request middleware failed: {0}")] - RequestMiddlewareError(#[from] reqwest_middleware::Error), - - #[error("parse url failed: {0}")] - UrlParseError(#[from] url::ParseError), - - #[error("parse JSON response failed: {0}")] - JsonParseError(#[from] serde_json::Error), - - #[error("invalid options: {0}")] - OptionsError(#[from] OptionsError), - - #[error("invalid runtime: {0}")] - CreateRuntimeError(#[from] CreateRuntimeError), - - #[error("invalid response")] - InvalidResponse, - - #[error("invalid stats: {0}")] - InvalidStats(String), -} - -impl From for ErrorReport { - fn from(value: NotionFdwError) -> Self { - ErrorReport::new(PgSqlErrorCode::ERRCODE_FDW_ERROR, format!("{value}"), "") - } -} - -type NotionFdwResult = Result; diff --git a/wrappers/src/fdw/notion_fdw/notion_fdw.rs b/wrappers/src/fdw/notion_fdw/notion_fdw.rs deleted file mode 100644 index 709e889d..00000000 --- a/wrappers/src/fdw/notion_fdw/notion_fdw.rs +++ /dev/null @@ -1,427 +0,0 @@ -use crate::stats; -use pgrx::{datum::datetime_support::to_timestamp, pg_sys, JsonB}; -use reqwest::{self, header, Url}; -use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; -use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; -use serde_json::{json, Value as JsonValue}; -use std::collections::HashMap; - -use super::{NotionFdwError, NotionFdwResult}; -use supabase_wrappers::prelude::*; - -// The construction of this fdw is heavily based on the Stripe FDW - -fn create_client(api_key: &str, notion_version: &str) -> NotionFdwResult { - let mut headers = header::HeaderMap::new(); - let value = format!("Bearer {}", api_key); - let mut auth_value = header::HeaderValue::from_str(&value)?; - auth_value.set_sensitive(true); - headers.insert(header::AUTHORIZATION, auth_value); - - let version_value = header::HeaderValue::from_str(notion_version)?; - headers.insert("Notion-Version", version_value); - - // Create the basic reqwest client - let reqwest_client = reqwest::Client::builder() - .default_headers(headers) - .build()?; - - // Get the retry policy - let retry_policy = ExponentialBackoff::builder().build_with_max_retries(3); - - // Create the client with middleware - let client = ClientBuilder::new(reqwest_client) - .with(RetryTransientMiddleware::new_with_policy(retry_policy)) - .build(); - - Ok(client) -} - -fn body_to_rows( - resp_body: &str, - cols: Vec<(&str, &str)>, - tgt_cols: &[Column], -) -> NotionFdwResult<(Vec, Option, Option)> { - let mut result = Vec::new(); - let value: JsonValue = serde_json::from_str(resp_body)?; - - // Notion API can return two types of responses: - // 1. Single Object Response: - // - The response is a single object where the "object" field specifies the object type (e.g., user, database). - // 2. List Response: - // - The response is a list of objects. The "object" field is set to 'list', and the "results" field contains the objects. The `type` field indicates the type of objects in the list (e.g., user, database). - - // Extract object type, can be either 'list' or a specific object type - let object: &str = value - .as_object() - .and_then(|v| v.get("object")) - .and_then(|v| v.as_str()) - .ok_or(NotionFdwError::InvalidResponse)?; - - // Handle single object response - let single_wrapped: Vec = match object { - "list" => Vec::new(), - _ => value - .as_object() - .map(|v| vec![JsonValue::Object(v.clone())]) - .ok_or(NotionFdwError::InvalidResponse)?, - }; - - // Then, get the list of entries - let entries = match object { - "list" => value - .as_object() - .and_then(|v| v.get("results")) - .and_then(|v| v.as_array()) - .ok_or(NotionFdwError::InvalidResponse)?, - _ => &single_wrapped, - }; - - for entry in entries { - let mut row = Row::new(); - - // Extract columns based on target columns specified - for tgt_col in tgt_cols { - // Extract the value of the target column - let tgt_col_value = entry - .as_object() - .and_then(|v| v.get(&tgt_col.name)) - .unwrap_or(&JsonValue::Null); - - // If we can't find the column type, default to jsonb - let col_type = cols - .iter() - .find(|(c, _)| c == &tgt_col.name) - .map(|(_, t)| t) - .unwrap_or(&"jsonb"); - - let cell = match *col_type { - "bool" => tgt_col_value.as_bool().map(Cell::Bool), - "i64" => tgt_col_value.as_i64().map(Cell::I64), - "string" => tgt_col_value.as_str().map(|a| Cell::String(a.to_owned())), - "timestamp" => tgt_col_value.as_i64().map(|a| { - let ts = to_timestamp(a as f64); - Cell::Timestamp(ts.to_utc()) - }), - "jsonb" => tgt_col_value - .as_object() - .map(|a| Cell::Json(JsonB(serde_json::Value::Object(a.clone())))), - _ => None, - }; - - row.push(tgt_col.name.as_str(), cell); - } - result.push(row); - } - - // Handle pagination - let cursor = value - .as_object() - .and_then(|v| v.get("next_cursor")) - .and_then(|v| v.as_str()) - .map(|s| s.to_owned()); - - let has_more = value - .as_object() - .and_then(|v| v.get("has_more")) - .and_then(|v| v.as_bool()); - - Ok((result, cursor, has_more)) -} - -fn pushdown_quals( - url: &mut Url, - _obj: &str, - quals: &[Qual], - fields: Vec<&str>, - page_size: i64, - cursor: &Option, -) { - // for scan with a single id query param, optimized to single object GET request - if quals.len() == 1 { - let qual = &quals[0]; - if qual.field == "id" && qual.operator == "=" && !qual.use_or { - if let Value::Cell(Cell::String(id)) = &qual.value { - let new_path = format!("{}/{}", url.path(), id); - url.set_path(&new_path); - url.set_query(None); - return; - } - } - } - - // pushdown quals - for qual in quals { - for field in &fields { - if qual.field == *field && qual.operator == "=" && !qual.use_or { - if let Value::Cell(cell) = &qual.value { - match cell { - Cell::Bool(b) => { - url.query_pairs_mut() - .append_pair(field, b.to_string().as_str()); - } - Cell::String(s) => { - url.query_pairs_mut().append_pair(field, s); - } - _ => {} - } - } - } - } - } - - // add pagination parameters except for 'balance' object - url.query_pairs_mut() - .append_pair("page_size", &format!("{}", page_size)); - if let Some(ref cursor) = cursor { - url.query_pairs_mut().append_pair("start_cursor", cursor); - } -} - -// get stats metadata -#[inline] -fn get_stats_metadata() -> JsonB { - stats::get_metadata(NotionFdw::FDW_NAME).unwrap_or(JsonB(json!({ - "request_cnt": 0i64, - }))) -} - -// save stats metadata -#[inline] -fn set_stats_metadata(stats_metadata: JsonB) { - stats::set_metadata(NotionFdw::FDW_NAME, Some(stats_metadata)); -} - -// increase stats metadata 'request_cnt' by 1 -#[inline] -fn inc_stats_request_cnt(stats_metadata: &mut JsonB) -> NotionFdwResult<()> { - if let Some(v) = stats_metadata.0.get_mut("request_cnt") { - *v = (v.as_i64().ok_or(NotionFdwError::InvalidStats( - "`request_cnt` is not a number".to_string(), - ))? + 1) - .into(); - }; - Ok(()) -} - -#[wrappers_fdw( - version = "0.1.0", - author = "Romain Graux", - website = "https://github.com/supabase/wrappers/tree/main/wrappers/src/fdw/notion_fdw", - error_type = "NotionFdwError" -)] -pub(crate) struct NotionFdw { - rt: Runtime, - base_url: Url, - client: Option, - scan_result: Option>, - iter_idx: usize, -} - -impl NotionFdw { - const FDW_NAME: &'static str = "NotionFdw"; - - fn build_url( - &self, - obj: &str, - quals: &[Qual], - page_size: i64, - cursor: &Option, - ) -> NotionFdwResult> { - let mut url = self.base_url.join(obj)?; - - // pushdown quals other than id - // ref: https://developers.notion.com/reference - let fields = match obj { - "users" => vec![], - _ => { - return Err(NotionFdwError::ObjectNotImplemented(obj.to_string())); - } - }; - pushdown_quals(&mut url, obj, quals, fields, page_size, cursor); - - Ok(Some(url)) - } - - fn resp_to_rows( - &self, - obj: &str, - resp_body: &str, - tgt_cols: &[Column], - ) -> NotionFdwResult<(Vec, Option, Option)> { - match obj { - "users" => body_to_rows( - resp_body, - vec![ - ("id", "string"), - ("type", "string"), - ("name", "string"), - ("avatar_url", "string"), - ("person", "jsonb"), - ("bot", "jsonb"), - ], - tgt_cols, - ), - _ => Err(NotionFdwError::ObjectNotImplemented(obj.to_string())), - } - } -} - -impl ForeignDataWrapper for NotionFdw { - fn new(options: &HashMap) -> NotionFdwResult { - let base_url = options - .get("api_url") - .map(|t| t.to_owned()) - // Ensure trailing slash is always present, otherwise /v1 will get obliterated when - // joined with object - .map(|s| { - if s.ends_with('/') { - s - } else { - format!("{}/", s) - } - }) - .unwrap_or_else(|| "https://api.notion.com/v1/".to_string()); - - let notion_version = options - .get("notion_version") - .map(|t| t.to_owned()) - .unwrap_or_else(|| "2022-06-28".to_string()); - - let client = match options.get("api_key") { - Some(api_key) => Some(create_client(api_key, ¬ion_version)), - None => { - let key_id = require_option("api_key_id", options)?; - get_vault_secret(key_id).map(|api_key| create_client(&api_key, ¬ion_version)) - } - } - .transpose()?; - - stats::inc_stats(Self::FDW_NAME, stats::Metric::CreateTimes, 1); - - Ok(NotionFdw { - rt: create_async_runtime()?, - base_url: Url::parse(&base_url)?, - client, - scan_result: None, - iter_idx: 0, - }) - } - - fn begin_scan( - &mut self, - quals: &[Qual], - columns: &[Column], - _sorts: &[Sort], - limit: &Option, - options: &HashMap, - ) -> NotionFdwResult<()> { - let obj = require_option("object", options)?; - - self.iter_idx = 0; - - if let Some(client) = &self.client { - let page_size = 100; // maximum page size limit for Notion API - let page_cnt = if let Some(limit) = limit { - if limit.count == 0 { - return Ok(()); - } - (limit.offset + limit.count) / page_size + 1 - } else { - // if no limit specified, fetch all records - i64::MAX - }; - let mut page = 0; - let mut result = Vec::new(); - let mut cursor: Option = None; - let mut stats_metadata = get_stats_metadata(); - - while page < page_cnt { - let url = self.build_url(obj, quals, page_size, &cursor)?; - let Some(url) = url else { - return Ok(()); - }; - - inc_stats_request_cnt(&mut stats_metadata)?; - - let body = self.rt.block_on(client.get(url).send()).and_then(|resp| { - stats::inc_stats( - Self::FDW_NAME, - stats::Metric::BytesIn, - resp.content_length().unwrap_or(0) as i64, - ); - - resp.error_for_status() - .and_then(|resp| self.rt.block_on(resp.text())) - .map_err(reqwest_middleware::Error::from) - })?; - - if body.is_empty() { - break; - } - - // convert response body to rows - let (rows, starting_after, has_more) = self.resp_to_rows(obj, &body, columns)?; - if rows.is_empty() { - break; - } - result.extend(rows); - match has_more { - Some(has_more) => { - if !has_more { - break; - } - // Otherwise, continue - } - None => break, - } - cursor = starting_after; - - page += 1; - } - - // save stats - stats::inc_stats(Self::FDW_NAME, stats::Metric::RowsIn, result.len() as i64); - stats::inc_stats(Self::FDW_NAME, stats::Metric::RowsOut, result.len() as i64); - set_stats_metadata(stats_metadata); - - self.scan_result = Some(result); - } - - Ok(()) - } - - fn iter_scan(&mut self, row: &mut Row) -> NotionFdwResult> { - if let Some(ref mut result) = self.scan_result { - if self.iter_idx < result.len() { - row.replace_with(result[self.iter_idx].clone()); - self.iter_idx += 1; - return Ok(Some(())); - } - } - Ok(None) - } - - fn re_scan(&mut self) -> NotionFdwResult<()> { - self.iter_idx = 0; - Ok(()) - } - - fn end_scan(&mut self) -> NotionFdwResult<()> { - self.scan_result.take(); - Ok(()) - } - - fn validator( - options: Vec>, - catalog: Option, - ) -> NotionFdwResult<()> { - if let Some(oid) = catalog { - if oid == FOREIGN_TABLE_RELATION_ID { - check_options_contain(&options, "object")?; - } - } - - Ok(()) - } -} diff --git a/wrappers/src/fdw/notion_fdw/tests.rs b/wrappers/src/fdw/notion_fdw/tests.rs deleted file mode 100644 index 2a2d889c..00000000 --- a/wrappers/src/fdw/notion_fdw/tests.rs +++ /dev/null @@ -1,85 +0,0 @@ -#[cfg(any(test, feature = "pg_test"))] -#[pgrx::pg_schema] -mod tests { - use pgrx::prelude::*; - - #[pg_test] - fn notion_smoketest() { - Spi::connect(|mut c| { - c.update( - r#"CREATE FOREIGN DATA WRAPPER notion_wrapper - HANDLER notion_fdw_handler VALIDATOR notion_fdw_validator"#, - None, - None, - ) - .unwrap(); - c.update( - r#"CREATE SERVER my_notion_server - FOREIGN DATA WRAPPER notion_wrapper - OPTIONS ( - api_url 'http://localhost:4242/v1', -- Notion API base URL, optional - notion_version '2021-08-23', -- Notion API version, optional - api_key 'sk_test_51LUmojFkiV6mfx3cpEzG9VaxhA86SA4DIj3b62RKHnRC0nhPp2JBbAmQ1izsX9RKD8rlzvw2xpY54AwZtXmWciif00Qi8J0w3O' -- Notion API Key, required - )"#, - None, - None, - ).unwrap(); - - c.update( - r#" - CREATE FOREIGN TABLE notion_users ( - name text, - type text, - id text - ) - SERVER my_notion_server - OPTIONS ( - object 'users' -- Corrected object name if previously incorrect - ) - "#, - None, - None, - ) - .unwrap(); - - let results = c - .select("SELECT name, type, id FROM notion_users", None, None) - .unwrap() - .map(|r| { - ( - r.get_by_name::<&str, _>("type").unwrap().unwrap(), - r.get_by_name::<&str, _>("name").unwrap().unwrap(), - r.get_by_name::<&str, _>("id").unwrap().unwrap(), - ) - }) - .collect::>(); - assert_eq!( - results, - vec![ - ("person", "John Doe", "d40e767c-d7af-4b18-a86d-55c61f1e39a4"), - ("bot", "Beep Boop", "9a3b5ae0-c6e6-482d-b0e1-ed315ee6dc57") - ] - ); - - let results = c - .select( - "SELECT * FROM notion_users WHERE id = 'd40e767c-d7af-4b18-a86d-55c61f1e39a4'", - None, - None, - ) - .unwrap() - .map(|r| { - ( - r.get_by_name::<&str, _>("type").unwrap().unwrap(), - r.get_by_name::<&str, _>("name").unwrap().unwrap(), - r.get_by_name::<&str, _>("id").unwrap().unwrap(), - ) - }) - .collect::>(); - assert_eq!( - results, - vec![("person", "John Doe", "d40e767c-d7af-4b18-a86d-55c61f1e39a4")] - ); - }); - } -} diff --git a/wrappers/src/fdw/wasm_fdw/tests.rs b/wrappers/src/fdw/wasm_fdw/tests.rs index 7db6175f..b177c7dc 100644 --- a/wrappers/src/fdw/wasm_fdw/tests.rs +++ b/wrappers/src/fdw/wasm_fdw/tests.rs @@ -107,6 +107,55 @@ mod tests { .filter_map(|r| r.get_by_name::<&str, _>("email").unwrap()) .collect::>(); assert_eq!(results, vec!["test@test.com"]); + + // Notion FDW test + c.update( + r#"CREATE SERVER notion_server + FOREIGN DATA WRAPPER wasm_wrapper + OPTIONS ( + fdw_package_url 'file://../../wasm-wrappers/fdw/notion_fdw/target/wasm32-unknown-unknown/release/notion_fdw.wasm', + fdw_package_name 'supabase:notion-fdw', + fdw_package_version '>=0.1.0', + api_url 'http://localhost:8096/notion', + api_key '1234567890' + )"#, + None, + None, + ) + .unwrap(); + c.update( + r#" + CREATE FOREIGN TABLE notion_pages ( + id text, + url text, + created_time timestamp, + last_edited_time timestamp, + archived boolean, + attrs jsonb + ) + SERVER notion_server + OPTIONS ( + object 'page' + ) + "#, + None, + None, + ) + .unwrap(); + + let results = c + .select( + "SELECT * FROM notion_pages WHERE id = '5a67c86f-d0da-4d0a-9dd7-f4cf164e6247'", + None, + None, + ) + .unwrap() + .filter_map(|r| r.get_by_name::<&str, _>("url").unwrap()) + .collect::>(); + assert_eq!( + results, + vec!["https://www.notion.so/test-page3-5a67c86fd0da4d0a9dd7f4cf164e6247"] + ); }); } }