diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties index 195f96502bf82..9ab25c390bc26 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties @@ -1 +1 @@ -version=0.4.7 +version=0.4.8 diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/PostgresTestDatabase.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/PostgresTestDatabase.java index 92ec49254a5bd..b52752f38b643 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/PostgresTestDatabase.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/PostgresTestDatabase.java @@ -11,20 +11,16 @@ import io.airbyte.cdk.db.factory.DatabaseDriver; import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.cdk.integrations.util.HostPortResolver; -import io.airbyte.commons.io.IOs; import io.airbyte.commons.string.Strings; import java.io.IOException; import java.io.UncheckedIOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.Arrays; import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.commons.io.FileUtils; import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.slf4j.Logger; @@ -42,6 +38,8 @@ */ public class PostgresTestDatabase implements AutoCloseable { + static private final Logger LOGGER = LoggerFactory.getLogger(PostgresTestDatabase.class); + /** * Create a new {@link PostgresTestDatabase} instance. * @@ -61,32 +59,15 @@ static public PostgresTestDatabase make(String imageName, String... methods) { private PostgresTestDatabase(PostgreSQLContainer sharedContainer) { this.container = sharedContainer; - this.suffix = Strings.addRandomSuffix("", "_", 10); - try { - this.tmpDir = Files.createTempDirectory("dir" + suffix); - } catch (final IOException e) { - throw new UncheckedIOException(e); - } - final var dir = this.tmpDir.toFile(); - Runtime.getRuntime().addShutdownHook(new Thread(() -> FileUtils.deleteQuietly(dir))); - this.dbName = "db" + suffix; this.userName = "test_user" + suffix; this.password = "test_password" + suffix; - - final Path script = this.tmpDir.resolve("create" + suffix + ".sql"); - IOs.writeFile(script, String.format(""" - CREATE DATABASE %s; - CREATE USER %s PASSWORD '%s'; - GRANT ALL PRIVILEGES ON DATABASE %s TO %s; - ALTER USER %s WITH SUPERUSER; - """, - dbName, - userName, password, - dbName, userName, - userName)); - PostgreSQLContainerHelper.runSqlScript(MountableFile.forHostPath(script), sharedContainer); + execSQL( + String.format("CREATE DATABASE %s", dbName), + String.format("CREATE USER %s PASSWORD '%s'", userName, password), + String.format("GRANT ALL PRIVILEGES ON DATABASE %s TO %s", dbName, userName), + String.format("ALTER USER %s WITH SUPERUSER", userName)); this.jdbcUrl = String.format( DatabaseDriver.POSTGRESQL.getUrlFormatString(), @@ -103,13 +84,10 @@ private PostgresTestDatabase(PostgreSQLContainer sharedContainer) { } public final PostgreSQLContainer container; - public final String dbName, userName, password, jdbcUrl; + public final String suffix, dbName, userName, password, jdbcUrl; public final DSLContext dslContext; public final Database database; - private final Path tmpDir; - private final String suffix; - /** * Convenience method for building identifiers which are unique to this instance. */ @@ -147,17 +125,38 @@ public PostgresUtils.Certificate getCertificate() { return new PostgresUtils.Certificate(caCert, clientCert, clientKey); } + private void execSQL(String... stmts) { + final List cmd = Stream.concat( + Stream.of("psql", "-a", "-d", container.getDatabaseName(), "-U", container.getUsername()), + Stream.of(stmts).flatMap(stmt -> Stream.of("-c", stmt))) + .toList(); + try { + LOGGER.debug("executing {}", Strings.join(cmd, " ")); + final var exec = container.execInContainer(cmd.toArray(new String[0])); + LOGGER.debug("exit code: {}\nstdout:\n{}\nstderr:\n{}", exec.getExitCode(), exec.getStdout(), exec.getStderr()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + /** + * Drop the database owned by this instance. + */ + public void dropDatabase() { + execSQL(String.format("DROP DATABASE %s", dbName)); + } + + /** + * Close resources held by this instance. This deliberately avoids dropping the database, which is + * really expensive in Postgres. This is because a DROP DATABASE in Postgres triggers a CHECKPOINT. + * Call {@link #dropDatabase} to explicitly drop the database. + */ @Override public void close() { dslContext.close(); - final Path script = this.tmpDir.resolve("drop" + suffix + ".sql"); - IOs.writeFile(script, String.format(""" - DROP USER %s; - DROP DATABASE %s; - """, - userName, - dbName)); - PostgreSQLContainerHelper.runSqlScript(MountableFile.forHostPath(script), container); + execSQL(String.format("DROP USER %s", userName)); } static private class ContainerFactory { diff --git a/airbyte-integrations/connectors/source-postgres/build.gradle b/airbyte-integrations/connectors/source-postgres/build.gradle index f806e1fdabb63..49e0bee537d6b 100644 --- a/airbyte-integrations/connectors/source-postgres/build.gradle +++ b/airbyte-integrations/connectors/source-postgres/build.gradle @@ -13,7 +13,7 @@ java { } airbyteJavaConnector { - cdkVersionRequired = '0.4.7' + cdkVersionRequired = '0.4.8' features = ['db-sources'] useLocalCdk = false } diff --git a/settings.gradle b/settings.gradle index 9cd532cbcbb5e..c7e237404a430 100644 --- a/settings.gradle +++ b/settings.gradle @@ -18,10 +18,13 @@ pluginManagement { // Configure the gradle enterprise plugin to enable build scans. Enabling the plugin at the top of the settings file allows the build scan to record // as much information as possible. plugins { - id "com.gradle.enterprise" version "3.13.1" + id "com.gradle.enterprise" version "3.15.1" id 'com.github.burrunan.s3-build-cache' version "1.5" } +ext.isCiServer = System.getenv().containsKey("CI") +ext.isAirbyteCI = System.getenv().containsKey("RUN_IN_AIRBYTE_CI") + dependencyResolutionManagement { // Set FAIL_ON_PROJECT_REPOS to ensure there are no more `repositories { ... }` blocks than necessary. repositoriesMode = RepositoriesMode.FAIL_ON_PROJECT_REPOS @@ -129,15 +132,13 @@ gradleEnterprise { buildScan { termsOfServiceUrl = "https://gradle.com/terms-of-service" termsOfServiceAgree = "yes" + uploadInBackground = (!isCiServer && !isAirbyteCI) // Disable in CI or scan URLs may not work. buildScanPublished { PublishedBuildScan scan -> file("scan-journal.log") << "${new Date()} - ${scan.buildScanId} - ${scan.buildScanUri}\n" } } } -ext.isCiServer = System.getenv().containsKey("CI") -ext.isAirbyteCI = System.getenv().containsKey("RUN_IN_AIRBYTE_CI") - if (isCiServer || isAirbyteCI) { buildCache { local {