From 529a734edc7eb1841fd5558fee923f2b68d1539b Mon Sep 17 00:00:00 2001 From: Marius Posta Date: Thu, 1 Feb 2024 11:41:29 -0500 Subject: [PATCH] remove source-mongodb-strict-encrypt --- .../build.gradle | 33 ---- .../MongodbSourceStrictEncrypt.java | 58 ------- ...godbSourceStrictEncryptAcceptanceTest.java | 157 ------------------ 3 files changed, 248 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-mongodb-strict-encrypt/build.gradle delete mode 100644 airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.source.mongodb/MongodbSourceStrictEncrypt.java delete mode 100644 airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongodbSourceStrictEncryptAcceptanceTest.java diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/build.gradle deleted file mode 100644 index 7d8895e4570f..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/build.gradle +++ /dev/null @@ -1,33 +0,0 @@ -plugins { - id 'application' - id 'airbyte-java-connector' -} - -airbyteJavaConnector { - cdkVersionRequired = '0.7.7' - features = ['db-sources'] - useLocalCdk = false -} - -//remove once upgrading the CDK version to 0.4.x or later -java { - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - -application { - mainClass = 'io.airbyte.integrations.source.mongodb.MongodbSourceStrictEncrypt' - applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] -} - -dependencies { - implementation project(':airbyte-integrations:connectors:source-mongodb-v2') - implementation project(':airbyte-integrations:connectors:destination-mongodb') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mongodb') - - - implementation 'org.mongodb:mongodb-driver-sync:4.3.0' -} diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.source.mongodb/MongodbSourceStrictEncrypt.java b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.source.mongodb/MongodbSourceStrictEncrypt.java deleted file mode 100644 index 834c97308143..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.source.mongodb/MongodbSourceStrictEncrypt.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mongodb; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.cdk.integrations.base.IntegrationRunner; -import io.airbyte.cdk.integrations.base.Source; -import io.airbyte.cdk.integrations.base.spec_modification.SpecModifyingSource; -import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.mongodb.MongoUtils; -import io.airbyte.integrations.destination.mongodb.MongoUtils.MongoInstanceType; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MongodbSourceStrictEncrypt extends SpecModifyingSource implements Source { - - private static final Logger LOGGER = LoggerFactory.getLogger(MongodbSourceStrictEncrypt.class); - - public MongodbSourceStrictEncrypt() { - super(new MongoDbSource()); - } - - @Override - public AirbyteConnectionStatus check(final JsonNode config) throws Exception { - final JsonNode instanceConfig = config.get(MongoUtils.INSTANCE_TYPE); - final MongoInstanceType instance = MongoInstanceType.fromValue(instanceConfig.get(MongoUtils.INSTANCE).asText()); - // If the MongoDb source connector is not set up to use a TLS connection, then we should fail the - // check. - if (instance.equals(MongoInstanceType.STANDALONE) && !MongoUtils.tlsEnabledForStandaloneInstance(config, instanceConfig)) { - throw new ConfigErrorException("TLS connection must be used to read from MongoDB."); - } - - return super.check(config); - } - - @Override - public ConnectorSpecification modifySpec(final ConnectorSpecification originalSpec) { - final ConnectorSpecification spec = Jsons.clone(originalSpec); - // removing tls property for a standalone instance to disable possibility to switch off a tls - // connection - ((ObjectNode) spec.getConnectionSpecification().get("properties").get("instance_type").get("oneOf").get(0).get("properties")).remove("tls"); - return spec; - } - - public static void main(final String[] args) throws Exception { - final Source source = new MongodbSourceStrictEncrypt(); - LOGGER.info("starting source: {}", MongodbSourceStrictEncrypt.class); - new IntegrationRunner(source).run(args); - LOGGER.info("completed source: {}", MongodbSourceStrictEncrypt.class); - } - -} diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongodbSourceStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongodbSourceStrictEncryptAcceptanceTest.java deleted file mode 100644 index 6e47cc8693bd..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongodbSourceStrictEncryptAcceptanceTest.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mongodb; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import static org.assertj.core.api.AssertionsForClassTypes.catchThrowable; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.mongodb.client.MongoCollection; -import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.destination.mongodb.MongoDatabase; -import io.airbyte.integrations.destination.mongodb.MongoUtils.MongoInstanceType; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.List; -import org.bson.BsonArray; -import org.bson.BsonString; -import org.bson.Document; -import org.junit.jupiter.api.Test; - -public class MongodbSourceStrictEncryptAcceptanceTest extends SourceAcceptanceTest { - - private static final String DATABASE_NAME = "test"; - private static final String COLLECTION_NAME = "acceptance_test1"; - private static final Path CREDENTIALS_PATH = Path.of("secrets/credentials.json"); - private static final String INSTANCE_TYPE = "instance_type"; - - protected JsonNode config; - protected MongoDatabase database; - - @Override - protected String getImageName() { - return "airbyte/source-mongodb-strict-encrypt:dev"; - } - - @Override - protected JsonNode getConfig() throws Exception { - return config; - } - - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - if (!Files.exists(CREDENTIALS_PATH)) { - throw new IllegalStateException( - "Must provide path to a MongoDB credentials file. By default {module-root}/" + CREDENTIALS_PATH - + ". Override by setting setting path with the CREDENTIALS_PATH constant."); - } - - config = Jsons.deserialize(Files.readString(CREDENTIALS_PATH)); - ((ObjectNode) config).put(JdbcUtils.DATABASE_KEY, DATABASE_NAME); - - final String connectionString = String.format("mongodb+srv://%s:%s@%s/%s?authSource=admin&retryWrites=true&w=majority&tls=true", - config.get("user").asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - config.get("instance_type").get("cluster_url").asText(), - config.get(JdbcUtils.DATABASE_KEY).asText()); - - database = new MongoDatabase(connectionString, DATABASE_NAME); - - final MongoCollection collection = database.createCollection(COLLECTION_NAME); - final var doc1 = new Document("id", "0001").append("name", "Test") - .append("test", 10).append("test_array", new BsonArray(List.of(new BsonString("test"), new BsonString("mongo")))) - .append("double_test", 100.12).append("int_test", 100); - final var doc2 = new Document("id", "0002").append("name", "Mongo").append("test", "test_value").append("int_test", 201); - final var doc3 = new Document("id", "0003").append("name", "Source").append("test", null) - .append("double_test", 212.11).append("int_test", 302); - - collection.insertMany(List.of(doc1, doc2, doc3)); - } - - @Override - protected void tearDown(final TestDestinationEnv testEnv) throws Exception { - for (final String collectionName : database.getCollectionNames()) { - database.getDatabase().getCollection(collectionName).drop(); - } - database.close(); - } - - @Override - protected ConnectorSpecification getSpec() throws Exception { - return Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class); - } - - @Override - protected ConfiguredAirbyteCatalog getConfiguredCatalog() throws Exception { - return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(Lists.newArrayList("_id")) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withCursorField(List.of("_id")) - .withStream(CatalogHelpers.createAirbyteStream( - DATABASE_NAME + "." + COLLECTION_NAME, - Field.of("_id", JsonSchemaType.STRING), - Field.of("id", JsonSchemaType.STRING), - Field.of("name", JsonSchemaType.STRING), - Field.of("test", JsonSchemaType.STRING), - Field.of("test_array", JsonSchemaType.ARRAY), - Field.of("empty_test", JsonSchemaType.STRING), - Field.of("double_test", JsonSchemaType.NUMBER), - Field.of("int_test", JsonSchemaType.NUMBER)) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.INCREMENTAL)) - .withDefaultCursorField(List.of("_id"))))); - } - - @Override - protected JsonNode getState() throws Exception { - return Jsons.jsonNode(new HashMap<>()); - } - - @Test - void testSpec() throws Exception { - final ConnectorSpecification actual = new MongodbSourceStrictEncrypt().spec(); - final ConnectorSpecification expected = getSpec(); - - assertEquals(expected, actual); - } - - @Test - void testCheck() throws Exception { - final JsonNode instanceConfig = Jsons.jsonNode(ImmutableMap.builder() - .put("instance", MongoInstanceType.STANDALONE.getType()) - .put("tls", false) - .build()); - - final JsonNode invalidStandaloneConfig = Jsons.clone(getConfig()); - - ((ObjectNode) invalidStandaloneConfig).put(INSTANCE_TYPE, instanceConfig); - - final Throwable throwable = catchThrowable(() -> new MongodbSourceStrictEncrypt().check(invalidStandaloneConfig)); - assertThat(throwable).isInstanceOf(ConfigErrorException.class); - assertThat(((ConfigErrorException) throwable) - .getDisplayMessage() - .contains("TLS connection must be used to read from MongoDB.")); - } - -}