diff --git a/build.gradle b/build.gradle index 7a11943cf8be..7a69fb68db47 100644 --- a/build.gradle +++ b/build.gradle @@ -952,6 +952,51 @@ project(':iceberg-snowflake') { } project(':iceberg-open-api') { + apply plugin: 'java-test-fixtures' + + dependencies { + testImplementation project(':iceberg-api') + testImplementation project(':iceberg-core') + testImplementation project(':iceberg-core').sourceSets.test.runtimeClasspath + testImplementation(testFixtures(project(':iceberg-open-api'))) + + testImplementation libs.junit.jupiter + testImplementation libs.junit.suite.api + testImplementation libs.junit.suite.engine + testImplementation libs.assertj.core + + testImplementation project(':iceberg-aws-bundle') + testImplementation project(':iceberg-gcp-bundle') + testImplementation project(':iceberg-azure-bundle') + + testFixturesImplementation project(':iceberg-api') + testFixturesImplementation project(':iceberg-core') + testFixturesImplementation project(path: ':iceberg-core', configuration: 'testArtifacts') + testFixturesImplementation project(':iceberg-core').sourceSets.test.runtimeClasspath + testFixturesImplementation project(':iceberg-aws') + testFixturesImplementation project(':iceberg-gcp') + testFixturesImplementation project(':iceberg-azure') + + testFixturesImplementation libs.jetty.servlet + testFixturesImplementation libs.jetty.server + testFixturesImplementation libs.sqlite.jdbc + } + + test { + useJUnitPlatform() + + // Always rerun the compatibility tests + outputs.upToDateWhen {false} + maxParallelForks = 1 + + // Pass through any system properties that start with "rck" (REST Compatibility Kit) + // Note: only pass through specific properties so they do not affect other build/test + // configurations + systemProperties System.properties + .findAll { k, v -> k.startsWith("rck") } + .collectEntries { k, v -> { [(k):v, (k.replaceFirst("rck.", "")):v] }} // strip prefix + } + def restCatalogSpec = "$projectDir/rest-catalog-open-api.yaml" tasks.register('validateRESTCatalogSpec', org.openapitools.generator.gradle.plugin.tasks.ValidateTask) { inputSpec.set(restCatalogSpec) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b8f20f3a2799..969ed7dad7da 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -62,6 +62,7 @@ jaxb-api = "2.3.1" jaxb-runtime = "2.3.9" jetty = "11.0.22" junit = "5.10.1" +junit-platform = "1.10.3" kafka = "3.8.0" kryo-shaded = "4.0.3" microprofile-openapi-api = "3.1.1" @@ -202,6 +203,8 @@ jetty-server = { module = "org.eclipse.jetty:jetty-server", version.ref = "jetty jetty-servlet = { module = "org.eclipse.jetty:jetty-servlet", version.ref = "jetty" } junit-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit" } junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "junit" } +junit-suite-api = { module = "org.junit.platform:junit-platform-suite-api", version.ref = "junit-platform" } +junit-suite-engine = { module = "org.junit.platform:junit-platform-suite-engine", version.ref = "junit-platform" } junit-vintage-engine = { module = "org.junit.vintage:junit-vintage-engine", version.ref = "junit" } kryo-shaded = { module = "com.esotericsoftware:kryo-shaded", version.ref = "kryo-shaded" } mockito-core = { module = "org.mockito:mockito-core", version.ref = "mockito" } diff --git a/open-api/README.md b/open-api/README.md index 6f906ec5acef..bff6d1dcab7e 100644 --- a/open-api/README.md +++ b/open-api/README.md @@ -40,3 +40,66 @@ make generate ``` The generated code is not being used in the project, but helps to see what the changes in the open-API definition are in the generated code. + +# REST Compatibility Kit (RCK) + +The REST Compatibility Kit (RCK) is a Technology Compatibility Kit (TCK) implementation for the +Iceberg REST Specification. This includes a series of tests based on the Java reference +implementation of the REST Catalog that can be executed against any REST server that implements the +spec. + +## Test Configuration + +The RCK can be configured using either environment variables or java system properties and allows +for configuring both the tests and the REST client. Environment variables prefixed by `CATALOG_` +are passed through the catalog configuring with the following mutations: + +1. The `CATALOG_` prefix is stripped from the key name +2. Single underscore (`_`) is replaced with a dot (`.`) +3. Double underscore (`__`) is replaced with a dash (`-`) +4. The key names are converted to lowercase + +A basic environment configuration would look like the following: + +```shell +CATALOG_URI=https://my_rest_server.io/ ## -> uri=https://my_rest_server.io/ +CATALOG_WAREHOUSE=test_warehouse ## -> warehouse=test_warehouse +CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO ## -> io-impl=org.apache.iceberg.aws.s3.S3FileIO +CATALOG_CREDENTIAL=: ## -> credential=: +``` + +Java properties pass to the test must be prefixed with `rck.`, which can be used to configure some +test configurations described below and any catalog client properties. + +An example of the same configuration using java system properties would look like the following: +```shell +rck.uri=https://my_rest_server.io/ ## -> uri=https://my_rest_server.io/ +rck.warehouse=test_warehouse ## -> warehouse=test_warehouse +rck.io-impl=org.apache.iceberg.aws.s3.S3FileIO ## -> io-impl=org.apache.iceberg.aws.s3.S3FileIO +rck.credential=: ## -> credential=: +``` + +Some test behaviors are configurable depending on the catalog implementations. Not all behaviors +are strictly defined by the REST Specification. The following are currently configurable: + +| config | default | +|-------------------------------|---------| +| rck.requires-namespace-create | true | +| rck.supports-serverside-retry | true | + + +## Running Compatibility Tests + +The compatibility tests can be invoked via gradle with the following: + +Note: The default behavior is to run a local http server with a jdbc backend for testing purposes, +so `-Drck.local=false` must be set to point to an external REST server. + +```shell +./gradlew :iceberg-open-api:test --tests RESTCompatibilityKitSuite \ + -Drck.local=false \ + -Drck.requires-namespace-create=true \ + -Drck.uri=https://my_rest_server.io/ \ + -Drck.warehouse=test_warehouse \ + -Drck.credential=: +``` \ No newline at end of file diff --git a/open-api/src/test/java/org/apache/iceberg/rest/RESTCompatibilityKitCatalogTests.java b/open-api/src/test/java/org/apache/iceberg/rest/RESTCompatibilityKitCatalogTests.java new file mode 100644 index 000000000000..f7224295ff26 --- /dev/null +++ b/open-api/src/test/java/org/apache/iceberg/rest/RESTCompatibilityKitCatalogTests.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.iceberg.catalog.CatalogTests; +import org.apache.iceberg.util.PropertyUtil; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@ExtendWith(RESTServerExtension.class) +public class RESTCompatibilityKitCatalogTests extends CatalogTests { + private static final Logger LOG = LoggerFactory.getLogger(RESTCompatibilityKitCatalogTests.class); + + private static RESTCatalog restCatalog; + + @BeforeAll + static void beforeClass() throws Exception { + restCatalog = RCKUtils.initCatalogClient(); + + assertThat(restCatalog.listNamespaces()) + .withFailMessage("Namespaces list should not contain: %s", RCKUtils.TEST_NAMESPACES) + .doesNotContainAnyElementsOf(RCKUtils.TEST_NAMESPACES); + } + + @BeforeEach + void before() { + try { + RCKUtils.purgeCatalogTestEntries(restCatalog); + } catch (Exception e) { + LOG.warn("Failure during test setup", e); + } + } + + @AfterAll + static void afterClass() throws Exception { + restCatalog.close(); + } + + @Override + protected RESTCatalog catalog() { + return restCatalog; + } + + @Override + protected boolean requiresNamespaceCreate() { + return PropertyUtil.propertyAsBoolean( + restCatalog.properties(), RCKUtils.RCK_REQUIRES_NAMESPACE_CREATE, false); + } + + @Override + protected boolean supportsServerSideRetry() { + return PropertyUtil.propertyAsBoolean( + restCatalog.properties(), RCKUtils.RCK_SUPPORTS_SERVERSIDE_RETRY, true); + } +} diff --git a/open-api/src/test/java/org/apache/iceberg/rest/RESTCompatibilityKitSuite.java b/open-api/src/test/java/org/apache/iceberg/rest/RESTCompatibilityKitSuite.java new file mode 100644 index 000000000000..3830dd8e60da --- /dev/null +++ b/open-api/src/test/java/org/apache/iceberg/rest/RESTCompatibilityKitSuite.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest; + +import org.junit.platform.suite.api.SelectClasses; +import org.junit.platform.suite.api.Suite; +import org.junit.platform.suite.api.SuiteDisplayName; + +/** + * Iceberg REST Compatibility Kit + * + *

This test suite provides the ability to run the Iceberg catalog tests against a remote REST + * catalog implementation to verify the behaviors against the reference implementation catalog + * tests. + * + *

The tests can be configured through environment variables or system properties. By default, + * the tests will run using a local http server using a servlet implementation that leverages the + * {@link RESTCatalogAdapter}. + */ +@Suite +@SuiteDisplayName("Iceberg REST Compatibility Kit") +@SelectClasses({RESTCompatibilityKitCatalogTests.class, RESTCompatibilityKitViewCatalogTests.class}) +public class RESTCompatibilityKitSuite {} diff --git a/open-api/src/test/java/org/apache/iceberg/rest/RESTCompatibilityKitViewCatalogTests.java b/open-api/src/test/java/org/apache/iceberg/rest/RESTCompatibilityKitViewCatalogTests.java new file mode 100644 index 000000000000..86fdb495cb82 --- /dev/null +++ b/open-api/src/test/java/org/apache/iceberg/rest/RESTCompatibilityKitViewCatalogTests.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.util.PropertyUtil; +import org.apache.iceberg.view.ViewCatalogTests; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@ExtendWith(RESTServerExtension.class) +public class RESTCompatibilityKitViewCatalogTests extends ViewCatalogTests { + private static final Logger LOG = + LoggerFactory.getLogger(RESTCompatibilityKitViewCatalogTests.class); + private static RESTCatalog restCatalog; + + @BeforeAll + static void beforeClass() throws Exception { + restCatalog = RCKUtils.initCatalogClient(); + + assertThat(restCatalog.listNamespaces()) + .withFailMessage("Namespaces list should not contain: %s", RCKUtils.TEST_NAMESPACES) + .doesNotContainAnyElementsOf(RCKUtils.TEST_NAMESPACES); + } + + @BeforeEach + void before() { + try { + RCKUtils.purgeCatalogTestEntries(restCatalog); + } catch (Exception e) { + LOG.warn("Failure during test setup", e); + } + } + + @AfterAll + static void afterClass() throws Exception { + restCatalog.close(); + } + + @Override + protected RESTCatalog catalog() { + return restCatalog; + } + + @Override + protected Catalog tableCatalog() { + return restCatalog; + } + + @Override + protected boolean requiresNamespaceCreate() { + return PropertyUtil.propertyAsBoolean( + restCatalog.properties(), RCKUtils.RCK_REQUIRES_NAMESPACE_CREATE, true); + } + + @Override + protected boolean supportsServerSideRetry() { + return PropertyUtil.propertyAsBoolean( + restCatalog.properties(), RCKUtils.RCK_SUPPORTS_SERVERSIDE_RETRY, true); + } + + @Override + protected boolean overridesRequestedLocation() { + return super.overridesRequestedLocation(); + } +} diff --git a/open-api/src/testFixtures/java/org/apache/iceberg/rest/RCKUtils.java b/open-api/src/testFixtures/java/org/apache/iceberg/rest/RCKUtils.java new file mode 100644 index 000000000000..ec9f7a91e70c --- /dev/null +++ b/open-api/src/testFixtures/java/org/apache/iceberg/rest/RCKUtils.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest; + +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.util.PropertyUtil; + +class RCKUtils { + private static final String CATALOG_ENV_PREFIX = "CATALOG_"; + static final String RCK_LOCAL = "rck.local"; + static final String RCK_REQUIRES_NAMESPACE_CREATE = "rck.requires-namespace-create"; + static final String RCK_SUPPORTS_SERVERSIDE_RETRY = "rck.supports-serverside-retry"; + static final String RCK_PURGE_TEST_NAMESPACES = "rck.purge-test-namespaces"; + + static final List TEST_NAMESPACES = List.of(Namespace.of("ns"), Namespace.of("newdb")); + + private RCKUtils() {} + + /** + * Utility method that allows configuring catalog properties via environment variables. + * + *

Returns a property map for all environment variables that start with CATALOG_ + * replacing double-underscore (__) with dash (-) and replacing single + * underscore (_) with dot (.) to allow for common catalog property + * conventions. All characters in the name are converted to lowercase and values are unmodified. + * + *

Examples: + * + * 


+   *     CATALOG_CATALOG__IMPL=org.apache.iceberg.jdbc.JdbcCatalog -> catalog-impl=org.apache.iceberg.jdbc.JdbcCatalog
+   *     CATALOG_URI=jdbc:sqlite:memory: -> uri=jdbc:sqlite:memory:
+   *     CATALOG_WAREHOUSE=test_warehouse -> warehouse=test_warehouse
+   *     CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO -> io-impl=org.apache.iceberg.aws.s3.S3FileIO
+   *     CATALOG_JDBC_USER=ice_user -> jdbc.user=ice_user
+   *
+ * + * @return configuration map + */ + static Map environmentCatalogConfig() { + return System.getenv().entrySet().stream() + .filter(e -> e.getKey().startsWith(CATALOG_ENV_PREFIX)) + .collect( + Collectors.toMap( + e -> + e.getKey() + .replaceFirst(CATALOG_ENV_PREFIX, "") + .replaceAll("__", "-") + .replaceAll("_", ".") + .toLowerCase(Locale.ROOT), + Map.Entry::getValue, + (m1, m2) -> { + throw new IllegalArgumentException("Duplicate key: " + m1); + }, + HashMap::new)); + } + + static RESTCatalog initCatalogClient() { + Map catalogProperties = Maps.newHashMap(); + catalogProperties.putAll(RCKUtils.environmentCatalogConfig()); + catalogProperties.putAll(Maps.fromProperties(System.getProperties())); + + // Set defaults + catalogProperties.putIfAbsent(CatalogProperties.URI, "http://localhost:8181/"); + catalogProperties.putIfAbsent(CatalogProperties.WAREHOUSE_LOCATION, "rck_warehouse"); + + RESTCatalog catalog = new RESTCatalog(); + catalog.setConf(new Configuration()); + catalog.initialize("rck_catalog", catalogProperties); + return catalog; + } + + static void purgeCatalogTestEntries(RESTCatalog catalog) { + if (!PropertyUtil.propertyAsBoolean(catalog.properties(), RCK_PURGE_TEST_NAMESPACES, true)) { + return; + } + + TEST_NAMESPACES.stream().filter(catalog::namespaceExists).forEach( + namespace -> { + catalog.listTables(namespace).forEach(catalog::dropTable); + catalog.listViews(namespace).forEach(catalog::dropView); + catalog.dropNamespace(namespace); + }); + } +} diff --git a/open-api/src/testFixtures/java/org/apache/iceberg/rest/RESTCatalogServer.java b/open-api/src/testFixtures/java/org/apache/iceberg/rest/RESTCatalogServer.java new file mode 100644 index 000000000000..dc79edb40378 --- /dev/null +++ b/open-api/src/testFixtures/java/org/apache/iceberg/rest/RESTCatalogServer.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest; + +import java.io.File; +import java.io.IOException; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.jdbc.JdbcCatalog; +import org.apache.iceberg.util.PropertyUtil; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.handler.gzip.GzipHandler; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.util.component.LifeCycle; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class RESTCatalogServer implements LifeCycle.Listener { + private static final Logger LOG = LoggerFactory.getLogger(RESTCatalogServer.class); + + static final String REST_PORT = "rest.port"; + static final int REST_PORT_DEFAULT = 8181; + + private Server httpServer; + + RESTCatalogServer() {} + + static class CatalogContext { + private final Catalog catalog; + private final Map configuration; + + CatalogContext(Catalog catalog, Map configuration) { + this.catalog = catalog; + this.configuration = configuration; + } + + public Catalog getCatalog() { + return catalog; + } + + public Map getConfiguration() { + return configuration; + } + } + + private CatalogContext initializeBackendCatalog() throws IOException { + // Translate environment variables to catalog properties + Map catalogProperties = RCKUtils.environmentCatalogConfig(); + + // Fallback to a JDBCCatalog impl if one is not set + catalogProperties.putIfAbsent(CatalogProperties.CATALOG_IMPL, JdbcCatalog.class.getName()); + catalogProperties.putIfAbsent(CatalogProperties.URI, "jdbc:sqlite::memory:"); + catalogProperties.putIfAbsent("jdbc.schema-version", "V1"); + + // Configure a default location if one is not specified + String warehouseLocation = catalogProperties.get(CatalogProperties.WAREHOUSE_LOCATION); + + if (warehouseLocation == null) { + File tmp = java.nio.file.Files.createTempDirectory("iceberg_warehouse").toFile(); + tmp.deleteOnExit(); + warehouseLocation = tmp.toPath().resolve("iceberg_data").toFile().getAbsolutePath(); + catalogProperties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouseLocation); + + LOG.info("No warehouse location set. Defaulting to temp location: {}", warehouseLocation); + } + + LOG.info("Creating catalog with properties: {}", catalogProperties); + return new CatalogContext( + CatalogUtil.buildIcebergCatalog("rest_backend", catalogProperties, new Configuration()), + catalogProperties); + } + + public void start(boolean join) throws Exception { + CatalogContext catalogContext = initializeBackendCatalog(); + + RESTCatalogAdapter adapter = new RESTServerCatalogAdapter(catalogContext); + RESTCatalogServlet servlet = new RESTCatalogServlet(adapter); + + ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); + context.setContextPath("/"); + ServletHolder servletHolder = new ServletHolder(servlet); + servletHolder.setInitParameter("javax.ws.rs.Application", "ServiceListPublic"); + context.addServlet(servletHolder, "/*"); + context.setVirtualHosts(null); + context.insertHandler(new GzipHandler()); + + this.httpServer = + new Server( + PropertyUtil.propertyAsInt(catalogContext.configuration, REST_PORT, REST_PORT_DEFAULT)); + httpServer.setHandler(context); + httpServer.start(); + + if(join) { + httpServer.join(); + } + } + + public void stop() throws Exception { + httpServer.stop(); + } + + public static void main(String[] args) throws Exception { + new RESTCatalogServer().start(true); + } +} diff --git a/open-api/src/testFixtures/java/org/apache/iceberg/rest/RESTServerCatalogAdapter.java b/open-api/src/testFixtures/java/org/apache/iceberg/rest/RESTServerCatalogAdapter.java new file mode 100644 index 000000000000..b90bad2d3ecc --- /dev/null +++ b/open-api/src/testFixtures/java/org/apache/iceberg/rest/RESTServerCatalogAdapter.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest; + +import java.util.Map; +import org.apache.iceberg.aws.s3.S3FileIOProperties; +import org.apache.iceberg.azure.AzureProperties; +import org.apache.iceberg.gcp.GCPProperties; +import org.apache.iceberg.rest.RESTCatalogServer.CatalogContext; +import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.apache.iceberg.util.PropertyUtil; + +class RESTServerCatalogAdapter extends RESTCatalogAdapter { + private static final String INCLUDE_CREDENTIALS = "include-credentials"; + + private final CatalogContext catalogContext; + + RESTServerCatalogAdapter(CatalogContext catalogContext) { + super(catalogContext.getCatalog()); + + this.catalogContext = catalogContext; + } + + @Override + public T handleRequest( + Route route, Map vars, Object body, Class responseType) { + T restResponse = super.handleRequest(route, vars, body, responseType); + + if (restResponse instanceof LoadTableResponse) { + if (PropertyUtil.propertyAsBoolean( + catalogContext.getConfiguration(), INCLUDE_CREDENTIALS, false)) { + applyCredentials( + catalogContext.getConfiguration(), ((LoadTableResponse) restResponse).config()); + } + } + + return restResponse; + } + + private void applyCredentials( + Map catalogConfig, Map tableConfig) { + if (catalogConfig.containsKey(S3FileIOProperties.ACCESS_KEY_ID)) { + tableConfig.put( + S3FileIOProperties.ACCESS_KEY_ID, catalogConfig.get(S3FileIOProperties.ACCESS_KEY_ID)); + } + + if (catalogConfig.containsKey(S3FileIOProperties.SECRET_ACCESS_KEY)) { + tableConfig.put( + S3FileIOProperties.SECRET_ACCESS_KEY, + catalogConfig.get(S3FileIOProperties.SECRET_ACCESS_KEY)); + } + + if (catalogConfig.containsKey(S3FileIOProperties.SESSION_TOKEN)) { + tableConfig.put( + S3FileIOProperties.SESSION_TOKEN, catalogConfig.get(S3FileIOProperties.SESSION_TOKEN)); + } + + if (catalogConfig.containsKey(GCPProperties.GCS_OAUTH2_TOKEN)) { + tableConfig.put( + GCPProperties.GCS_OAUTH2_TOKEN, catalogConfig.get(GCPProperties.GCS_OAUTH2_TOKEN)); + } + + catalogConfig.entrySet().stream() + .filter( + entry -> + entry.getKey().startsWith(AzureProperties.ADLS_SAS_TOKEN_PREFIX) + || entry.getKey().startsWith(AzureProperties.ADLS_CONNECTION_STRING_PREFIX)) + .forEach(entry -> tableConfig.put(entry.getKey(), entry.getValue())); + } +} diff --git a/open-api/src/testFixtures/java/org/apache/iceberg/rest/RESTServerExtension.java b/open-api/src/testFixtures/java/org/apache/iceberg/rest/RESTServerExtension.java new file mode 100644 index 000000000000..14a8ed73c958 --- /dev/null +++ b/open-api/src/testFixtures/java/org/apache/iceberg/rest/RESTServerExtension.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest; + +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; + +public class RESTServerExtension implements BeforeAllCallback, AfterAllCallback { + private RESTCatalogServer localServer; + + @Override + public void beforeAll(ExtensionContext extensionContext) throws Exception { + if (Boolean.parseBoolean( + extensionContext + .getConfigurationParameter(RCKUtils.RCK_LOCAL) + .orElse("true"))) { + this.localServer = new RESTCatalogServer(); + this.localServer.start(false); + } + } + + @Override + public void afterAll(ExtensionContext extensionContext) throws Exception { + if (localServer != null) { + localServer.stop(); + } + } +}