From 56d65e0431ca9e476b1d522787231e153cb88b3e Mon Sep 17 00:00:00 2001 From: IrisWan <150207222+WanYixian@users.noreply.github.com> Date: Thu, 2 Jan 2025 16:57:20 +0800 Subject: [PATCH] Update Iceberg rest catalog (#166) * iceberg rest catalog * minor update * Update apache-iceberg.mdx --- integrations/destinations/apache-iceberg.mdx | 26 +++++++++++++------- integrations/sources/apache-iceberg.mdx | 26 +++++++++++++------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/integrations/destinations/apache-iceberg.mdx b/integrations/destinations/apache-iceberg.mdx index e1680f52..660b5777 100644 --- a/integrations/destinations/apache-iceberg.mdx +++ b/integrations/destinations/apache-iceberg.mdx @@ -44,6 +44,10 @@ WITH ( | catalog.url | Conditional. The URL of the catalog. It is required when catalog.type is not storage. | | primary\_key | The primary key for an upsert sink. It is only applicable to the upsert mode. | | commit\_checkpoint\_interval | Optional. Commit every N checkpoints (N > 0). Default value is 10.
The behavior of this field also depends on the `sink_decouple` setting:| +| catalog.credential | Optional. Credential for accessing the Iceberg catalog, used to exchange for a token in the OAuth2 client credentials flow. Applicable only in the `rest` catalog.| +| catalog.token| Optional. A Bearer token for accessing the Iceberg catalog, used for interaction with the server. Applicable only in the `rest` catalog. | +| catalog.oauth2-server-uri | Optional. The `oauth2-server-uri` for accessing the Iceberg catalog, serving as the token endpoint URI to fetch a token if the `rest` catalog is not the authorization server. Applicable only in the `rest` catalog.| +| catalog.scope | Optional. Scope for accessing the Iceberg catalog, providing additional scope for OAuth2. Applicable only in the `rest` catalog. | ## Data type mapping @@ -101,16 +105,20 @@ with ( connector = 'iceberg', type = 'append-only', force_append_only = true, - s3.endpoint = 'http://minio-0:9301', - s3.access.key = 'xxxxxxxxxx', - s3.secret.key = 'xxxxxxxxxx', - s3.region = 'ap-southeast-1', + s3.endpoint = 'https://s3.ap-southeast-2.amazonaws.com', + s3.region = 'ap-southeast-2', + s3.access.key = 'xxxx', + s3.secret.key = 'xxxx', + s3.path.style.access = 'true', catalog.type = 'rest', - catalog.name = 'demo', - catalog.uri = 'http://rest:8181', - warehouse.path = 's3://icebergdata/demo', - database.name = 's1', - table.name = 't1' + catalog.uri = 'http://localhost:8181/api/catalog', + warehouse.path = 'quickstart_catalog', + database.name = 'ns', + table.name = 't1', + catalog.credential='123456:123456', + catalog.scope='PRINCIPAL_ROLE:ALL', + catalog.oauth2-server-uri='xxx' + catalog.scope='xxx', ); ``` diff --git a/integrations/sources/apache-iceberg.mdx b/integrations/sources/apache-iceberg.mdx index 24c756d4..e20cefec 100644 --- a/integrations/sources/apache-iceberg.mdx +++ b/integrations/sources/apache-iceberg.mdx @@ -39,6 +39,10 @@ You don’t need to specify the column name for the Iceberg source, as RisingWav | catalog.type | Optional. The catalog type used in this table. Currently, the supported values are `storage`, `rest`, `hive`, `jdbc`, and `glue`. If not specified, `storage` is used. For details, see [Catalogs](#catalogs). | | warehouse.path | Conditional. The path of the Iceberg warehouse. Currently, only S3-compatible object storage systems, such as AWS S3 and MinIO, are supported. It's required if the `catalog.type` is not `rest`. | | catalog.url | Conditional. The URL of the catalog. It is required when `catalog.type` is not `storage`. | +| catalog.credential | Optional. Credential for accessing the Iceberg catalog, used to exchange for a token in the OAuth2 client credentials flow. Applicable only in the `rest` catalog.| +| catalog.token| Optional. A Bearer token for accessing the Iceberg catalog, used for interaction with the server. Applicable only in the `rest` catalog. | +| catalog.oauth2-server-uri | Optional. The `oauth2-server-uri` for accessing the Iceberg catalog, serving as the token endpoint URI to fetch a token if the `rest` catalog is not the authorization server. Applicable only in the `rest` catalog.| +| catalog.scope | Optional. Scope for accessing the Iceberg catalog, providing additional scope for OAuth2. Applicable only in the `rest` catalog. | ## Data type mapping @@ -83,16 +87,20 @@ Examples create source source_demo_rest with ( connector = 'iceberg', - s3.endpoint = 'http://minio-0:9301', - s3.access.key = 'xxxxxxxxxx', - s3.secret.key = 'xxxxxxxxxx', - s3.region = 'ap-southeast-1', + s3.endpoint = 'https://s3.ap-southeast-2.amazonaws.com', + s3.region = 'ap-southeast-2', + s3.access.key = 'xxxx', + s3.secret.key = 'xxxx', + s3.path.style.access = 'true', catalog.type = 'rest', - catalog.name = 'demo', - catalog.uri = 'http://rest:8181', - warehouse.path = 's3://icebergdata/demo', - database.name = 's1', - table.name = 't1' + catalog.uri = 'http://localhost:8181/api/catalog', + warehouse.path = 'quickstart_catalog', + database.name = 'ns', + table.name = 't1', + catalog.credential='123456:123456', + catalog.scope='PRINCIPAL_ROLE:ALL', + catalog.oauth2-server-uri='xxx' + catalog.scope='xxx', ); ``` * Hive catalog: RisingWave supports the Hive catalog. You need to set `catalog.type` to `hive` to use it.