diff --git a/extension/persistence/eclipselink/build.gradle.kts b/extension/persistence/eclipselink/build.gradle.kts index 12e0affd0..3137a0cc0 100644 --- a/extension/persistence/eclipselink/build.gradle.kts +++ b/extension/persistence/eclipselink/build.gradle.kts @@ -44,7 +44,7 @@ tasks.register("archiveConf") { archiveFileName = "conf.jar" destinationDirectory = layout.buildDirectory.dir("conf") - from("src/test/resources/META-INF/") { include("persistence.xml") } + from("src/main/resources/META-INF/") { include("persistence.xml") } } tasks.named("test") { dependsOn("archiveConf") } diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index dec10d3a4..da6d90055 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -111,7 +111,8 @@ public PolarisEclipseLinkMetaStoreSessionImpl( @NotNull RealmContext realmContext, @Nullable String confFile, @Nullable String persistenceUnitName) { - LOGGER.debug("Create EclipseLink Meta Store Session for {}", realmContext.getRealmIdentifier()); + LOGGER.debug( + "Creating EclipseLink Meta Store Session for realm {}", realmContext.getRealmIdentifier()); emf = createEntityManagerFactory(realmContext, confFile, persistenceUnitName); // init store @@ -153,8 +154,8 @@ private EntityManagerFactory createEntityManagerFactory( prefixUrl = new File(jarPrefixPath).toURI().toURL(); } - LOGGER.info( - "Created a new ClassLoader with the jar {} in classpath to load the config file", + LOGGER.debug( + "Creating a new ClassLoader with the jar {} in classpath to load the config file", prefixUrl); URLClassLoader currentClassLoader = diff --git a/extension/persistence/eclipselink/src/main/resources/META-INF/persistence.xml b/extension/persistence/eclipselink/src/main/resources/META-INF/persistence.xml index a364fe06c..913713796 100644 --- a/extension/persistence/eclipselink/src/main/resources/META-INF/persistence.xml +++ b/extension/persistence/eclipselink/src/main/resources/META-INF/persistence.xml @@ -35,7 +35,7 @@ NONE + value="jdbc:h2:file:./build/test_data/polaris/{realm}/db"/> diff --git a/extension/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java b/extension/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java index ea28f8c67..000a909e9 100644 --- a/extension/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java +++ b/extension/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java @@ -51,7 +51,7 @@ protected PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() { PolarisEclipseLinkStore store = new PolarisEclipseLinkStore(diagServices); PolarisEclipseLinkMetaStoreSessionImpl session = new PolarisEclipseLinkMetaStoreSessionImpl( - store, Mockito.mock(), () -> "realm", null, "polaris-dev"); + store, Mockito.mock(), () -> "realm", null, "polaris"); return new PolarisTestMetaStoreManager( new PolarisMetaStoreManagerImpl(), new PolarisCallContext( @@ -72,7 +72,7 @@ void testCreateStoreSession(String confFile, boolean success) { try { var session = new PolarisEclipseLinkMetaStoreSessionImpl( - store, Mockito.mock(), () -> "realm", confFile, "polaris-dev"); + store, Mockito.mock(), () -> "realm", confFile, "polaris"); assertNotNull(session); assertTrue(success); } catch (Exception e) { diff --git a/extension/persistence/eclipselink/src/test/resources/META-INF/persistence.xml b/extension/persistence/eclipselink/src/test/resources/META-INF/persistence.xml deleted file mode 100644 index 64952f7c0..000000000 --- a/extension/persistence/eclipselink/src/test/resources/META-INF/persistence.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - - - - org.eclipse.persistence.jpa.PersistenceProvider - org.apache.polaris.core.persistence.models.ModelEntity - org.apache.polaris.core.persistence.models.ModelEntityActive - org.apache.polaris.core.persistence.models.ModelEntityChangeTracking - org.apache.polaris.core.persistence.models.ModelEntityDropped - org.apache.polaris.core.persistence.models.ModelGrantRecord - org.apache.polaris.core.persistence.models.ModelPrincipalSecrets - org.apache.polaris.core.persistence.models.ModelSequenceId - NONE - - - - - - - - - - - \ No newline at end of file diff --git a/polaris-service/build.gradle.kts b/polaris-service/build.gradle.kts index c0cfa60f7..6363842b2 100644 --- a/polaris-service/build.gradle.kts +++ b/polaris-service/build.gradle.kts @@ -109,6 +109,9 @@ dependencies { testImplementation(libs.assertj.core) testImplementation(libs.mockito.core) testRuntimeOnly("org.junit.platform:junit-platform-launcher") + + testRuntimeOnly(project(":polaris-eclipselink")) + testRuntimeOnly(libs.h2) } if (project.properties.get("eclipseLink") == "true") { diff --git a/polaris-service/src/main/java/org/apache/polaris/service/BootstrapRealmsCommand.java b/polaris-service/src/main/java/org/apache/polaris/service/BootstrapRealmsCommand.java index 3e7b8fc31..aab0e1229 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/BootstrapRealmsCommand.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/BootstrapRealmsCommand.java @@ -48,8 +48,7 @@ public BootstrapRealmsCommand() { protected void run( Bootstrap bootstrap, Namespace namespace, - PolarisApplicationConfig configuration) - throws Exception { + PolarisApplicationConfig configuration) { MetaStoreManagerFactory metaStoreManagerFactory = configuration.getMetaStoreManagerFactory(); PolarisConfigurationStore configurationStore = configuration.getConfigurationStore(); diff --git a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingCatalogTest.java index 59842b6c8..4f6400864 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingCatalogTest.java @@ -28,6 +28,7 @@ import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.Response; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -60,9 +61,12 @@ public class PolarisOverlappingCatalogTest { private static String realm; @BeforeAll - public static void setup(PolarisConnectionExtension.PolarisToken adminToken) { + public static void setup(PolarisConnectionExtension.PolarisToken adminToken) throws IOException { userToken = adminToken.token(); - realm = PolarisConnectionExtension.getTestRealm(PolarisServiceImplIntegrationTest.class); + realm = PolarisConnectionExtension.getTestRealm(PolarisOverlappingCatalogTest.class); + + // Set up the database location + PolarisConnectionExtension.createTestDir(realm); } private Response createCatalog(String prefix, String defaultBaseLocation, boolean isExternal) { diff --git a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplIntegrationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplIntegrationTest.java index 22a4c42b3..dce76c18a 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplIntegrationTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplIntegrationTest.java @@ -99,9 +99,12 @@ public class PolarisServiceImplIntegrationTest { private static String realm; @BeforeAll - public static void setup(PolarisConnectionExtension.PolarisToken adminToken) { + public static void setup(PolarisConnectionExtension.PolarisToken adminToken) throws IOException { userToken = adminToken.token(); realm = PolarisConnectionExtension.getTestRealm(PolarisServiceImplIntegrationTest.class); + + // Set up test location + PolarisConnectionExtension.createTestDir(realm); } @AfterEach diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java index b2ccb992c..ac1d5f2c0 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java @@ -31,14 +31,11 @@ import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.Response; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; -import org.apache.commons.io.FileUtils; import org.apache.iceberg.BaseTable; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.Schema; @@ -133,9 +130,8 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests public static void setup() throws IOException { realm = PolarisConnectionExtension.getTestRealm(PolarisRestCatalogIntegrationTest.class); - Path testDir = Path.of("build/test_data/iceberg/" + realm); - FileUtils.deleteQuietly(testDir.toFile()); - Files.createDirectories(testDir); + // Set up test location + PolarisConnectionExtension.createTestDir(realm); } @BeforeEach diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewIntegrationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewIntegrationTest.java index 9b557bdda..b2b69a831 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewIntegrationTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewIntegrationTest.java @@ -29,11 +29,8 @@ import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.Response; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.List; import java.util.Optional; -import org.apache.commons.io.FileUtils; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.catalog.SessionCatalog; import org.apache.iceberg.rest.HTTPClient; @@ -96,9 +93,8 @@ public class PolarisRestCatalogViewIntegrationTest extends ViewCatalogTests "realmName"; + } @Test public void testCleanupFileNotExists() throws IOException { - InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - RealmContext realmContext = () -> "realmName"; PolarisCallContext polarisCallContext = new PolarisCallContext( metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), @@ -83,9 +89,6 @@ public void testCleanupFileNotExists() throws IOException { @Test public void testCleanupFileManifestExistsDataFilesDontExist() throws IOException { - InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - RealmContext realmContext = () -> "realmName"; PolarisCallContext polarisCallContext = new PolarisCallContext( metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), @@ -116,9 +119,6 @@ public void testCleanupFileManifestExistsDataFilesDontExist() throws IOException @Test public void testCleanupFiles() throws IOException { - InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - RealmContext realmContext = () -> "realmName"; PolarisCallContext polarisCallContext = new PolarisCallContext( metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), @@ -166,9 +166,6 @@ public void close() { @Test public void testCleanupFilesWithRetries() throws IOException { - InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - RealmContext realmContext = () -> "realmName"; PolarisCallContext polarisCallContext = new PolarisCallContext( metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), diff --git a/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java b/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java index d106a26e0..ab9f9324c 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java @@ -41,17 +41,23 @@ import org.apache.polaris.core.entity.TaskEntity; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.slf4j.LoggerFactory; class TableCleanupTaskHandlerTest { + private InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory; + private RealmContext realmContext; + + @BeforeEach + void setUp() { + metaStoreManagerFactory = new InMemoryPolarisMetaStoreManagerFactory(); + realmContext = () -> "realmName"; + } @Test public void testTableCleanup() throws IOException { - InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - RealmContext realmContext = () -> "realmName"; PolarisCallContext polarisCallContext = new PolarisCallContext( metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), @@ -112,9 +118,6 @@ public void testTableCleanup() throws IOException { @Test public void testTableCleanupHandlesAlreadyDeletedMetadata() throws IOException { - InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - RealmContext realmContext = () -> "realmName"; PolarisCallContext polarisCallContext = new PolarisCallContext( metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), @@ -174,9 +177,6 @@ public void close() { @Test public void testTableCleanupDuplicatesTasksIfFileStillExists() throws IOException { - InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - RealmContext realmContext = () -> "realmName"; PolarisCallContext polarisCallContext = new PolarisCallContext( metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), @@ -269,9 +269,6 @@ public void close() { @Test public void testTableCleanupMultipleSnapshots() throws IOException { - InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - RealmContext realmContext = () -> "realmName"; PolarisCallContext polarisCallContext = new PolarisCallContext( metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), diff --git a/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisConnectionExtension.java b/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisConnectionExtension.java index bc267c7eb..bc181a665 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisConnectionExtension.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisConnectionExtension.java @@ -24,9 +24,14 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import io.dropwizard.testing.junit5.DropwizardAppExtension; +import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Modifier; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; +import java.util.Comparator; +import java.util.List; import java.util.Map; import java.util.Optional; import org.apache.polaris.core.context.CallContext; @@ -40,7 +45,9 @@ import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.service.auth.TokenUtils; import org.apache.polaris.service.config.PolarisApplicationConfig; +import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.jetbrains.annotations.Nullable; +import org.junit.jupiter.api.extension.AfterAllCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.ExtensionContext.Namespace; @@ -50,7 +57,8 @@ import org.junit.platform.commons.util.ReflectionUtils; import org.slf4j.LoggerFactory; -public class PolarisConnectionExtension implements BeforeAllCallback, ParameterResolver { +public class PolarisConnectionExtension + implements BeforeAllCallback, AfterAllCallback, ParameterResolver { public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private MetaStoreManagerFactory metaStoreManagerFactory; @@ -78,6 +86,9 @@ public void beforeAll(ExtensionContext extensionContext) throws Exception { PolarisApplicationConfig config = (PolarisApplicationConfig) dropwizardAppExtension.getConfiguration(); metaStoreManagerFactory = config.getMetaStoreManagerFactory(); + if (!(metaStoreManagerFactory instanceof InMemoryPolarisMetaStoreManagerFactory)) { + metaStoreManagerFactory.bootstrapRealms(List.of(realm)); + } RealmContext realmContext = config @@ -109,10 +120,40 @@ public void beforeAll(ExtensionContext extensionContext) throws Exception { } } + @Override + public void afterAll(ExtensionContext context) { + if (!(metaStoreManagerFactory instanceof InMemoryPolarisMetaStoreManagerFactory)) { + metaStoreManagerFactory.purgeRealms(List.of(realm)); + } + } + public static String getTestRealm(Class testClassName) { return testClassName.getName().replace('.', '_'); } + public static void createTestDir(String realm) throws IOException { + // Set up the database location + Path testDir = Path.of("build/test_data/polaris/" + realm); + if (Files.exists(testDir)) { + if (Files.isDirectory(testDir)) { + Files.walk(testDir) + .sorted(Comparator.reverseOrder()) + .forEach( + path -> { + try { + Files.delete(path); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + } else { + Files.delete(testDir); + } + } + Files.createDirectories(testDir); + } + static PolarisPrincipalSecrets getAdminSecrets() { return adminSecrets; } @@ -137,7 +178,11 @@ static PolarisPrincipalSecrets getAdminSecrets() { public boolean supportsParameter( ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { - return parameterContext.getParameter().getType().equals(PolarisToken.class) + return parameterContext + .getParameter() + .getType() + .equals(PolarisConnectionExtension.PolarisToken.class) + || parameterContext.getParameter().getType().equals(MetaStoreManagerFactory.class) || parameterContext.getParameter().getType().equals(PolarisPrincipalSecrets.class); } diff --git a/polaris-service/src/test/resources/polaris-server-integrationtest.yml b/polaris-service/src/test/resources/polaris-server-integrationtest.yml index fd4a9d53e..ba8960669 100644 --- a/polaris-service/src/test/resources/polaris-server-integrationtest.yml +++ b/polaris-service/src/test/resources/polaris-server-integrationtest.yml @@ -83,7 +83,7 @@ metaStoreManager: # type: remote # url: http://sdp-devvm-mcollado:8080 # type: eclipse-link # uncomment to use eclipse-link as metastore -# persistence-unit: polaris-dev +# persistence-unit: polaris io: factoryType: default