From cdb1238a5ef586c882347f78b4e896cd7bb02a95 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 24 Jan 2024 12:59:18 -0800 Subject: [PATCH 01/96] ... --- .../base/TypingAndDedupingFlag.java | 4 ++ .../src/main/resources/version.properties | 2 +- .../airbyte-cdk/db-destinations/build.gradle | 3 + .../jdbc/AbstractJdbcDestination.java | 3 +- .../typing_deduping/RawOnlySqlGenerator.kt | 60 ++++++++++++++++ .../typing_deduping/NoOpSqlGenerator.java | 69 +++++++++++++++++++ .../destination-clickhouse/build.gradle | 6 +- .../clickhouse/ClickhouseDestination.java | 8 ++- .../clickhouse/ClickhouseSqlOperations.java | 10 ++- .../src/main/resources/spec.json | 13 ++++ 10 files changed, 170 insertions(+), 8 deletions(-) create mode 100644 airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt create mode 100644 airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpSqlGenerator.java diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/TypingAndDedupingFlag.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/TypingAndDedupingFlag.java index b59e757b62c4..c15e5d85362a 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/TypingAndDedupingFlag.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/TypingAndDedupingFlag.java @@ -21,4 +21,8 @@ public static Optional getRawNamespaceOverride(String option) { } } + public static boolean isUsingV2RawTable() { + return isDestinationV2() || DestinationConfig.getInstance().getBooleanValue("use_v2_raw_table"); + } + } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties index b18dfa7feb69..a4868348559c 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties @@ -1 +1 @@ -version=0.13.2 +version=0.13.3 diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle index 4668be6e7cca..b83efde9dde3 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle @@ -1,3 +1,6 @@ +plugins { + id "org.jetbrains.kotlin.jvm" version "1.9.22" +} java { compileJava { diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java index d25b6ecb4296..0741f441287f 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java @@ -288,6 +288,7 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN } } final JdbcSqlGenerator sqlGenerator = getSqlGenerator(); + // SqlGenerator should be ok here because the catalog parser just uses "buildStreamid" final ParsedCatalog parsedCatalog = TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE) .map(override -> new CatalogParser(sqlGenerator, override)) .orElse(new CatalogParser(sqlGenerator)) @@ -296,7 +297,7 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN final var migrator = new JdbcV1V2Migrator(namingResolver, database, databaseName); final NoopV2TableMigrator v2TableMigrator = new NoopV2TableMigrator(); final DestinationHandler destinationHandler = getDestinationHandler(databaseName, database); - final boolean disableTypeDedupe = config.has(DISABLE_TYPE_DEDUPE) && config.get(DISABLE_TYPE_DEDUPE).asBoolean(false); + final boolean disableTypeDedupe = !config.has(DISABLE_TYPE_DEDUPE) || config.get(DISABLE_TYPE_DEDUPE).asBoolean(false); final TyperDeduper typerDeduper; if (disableTypeDedupe) { typerDeduper = new NoOpTyperDeduperWithV1V2Migrations<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator, diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt new file mode 100644 index 000000000000..7994a22e2923 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt @@ -0,0 +1,60 @@ +package io.airbyte.cdk.integrations.destination.jdbc.typing_deduping + +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer +import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType +import io.airbyte.integrations.base.destination.typing_deduping.ColumnId +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig +import org.jooq.Condition +import org.jooq.DataType +import org.jooq.Field +import org.jooq.SQLDialect +import java.util.* + +class RawOnlySqlGenerator(namingTransformer: NamingConventionTransformer?) : + JdbcSqlGenerator(namingTransformer) { + override fun getStructType(): DataType<*>? { + return null + } + + override fun getArrayType(): DataType<*>? { + return null + } + + override fun getWidestType(): DataType<*>? { + return null + } + + override fun getDialect(): SQLDialect? { + return null + } + + override fun extractRawDataFields( + columns: LinkedHashMap, + useExpensiveSaferCasting: Boolean + ): List>? { + return null + } + + override fun buildAirbyteMetaColumn(columns: LinkedHashMap): Field<*>? { + return null + } + + override fun cdcDeletedAtNotNullCondition(): Condition? { + return null + } + + override fun getRowNumber( + primaryKey: List, + cursorField: Optional + ): Field? { + return null + } + + override fun existingSchemaMatchesStreamConfig( + stream: StreamConfig, + existingTable: TableDefinition + ): Boolean { + return false + } +} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpSqlGenerator.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpSqlGenerator.java new file mode 100644 index 000000000000..6a423fcb018c --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpSqlGenerator.java @@ -0,0 +1,69 @@ +package io.airbyte.integrations.base.destination.typing_deduping; + +import java.util.Optional; +import lombok.NoArgsConstructor; + +@NoArgsConstructor +public class NoOpSqlGenerator implements SqlGenerator { + + @Override + public StreamId buildStreamId(final String namespace, final String name, final String rawNamespaceOverride) { + return null; + } + + @Override + public ColumnId buildColumnId(final String name) { + return SqlGenerator.super.buildColumnId(name); + } + + @Override + public ColumnId buildColumnId(final String name, final String suffix) { + return null; + } + + @Override + public Sql createTable(final StreamConfig stream, final String suffix, final boolean force) { + return null; + } + + @Override + public Sql createSchema(final String schema) { + return null; + } + + @Override + public boolean existingSchemaMatchesStreamConfig(final StreamConfig stream, final Object existingTable) { + return false; + } + + @Override + public Sql updateTable(final StreamConfig stream, final String finalSuffix, final Optional minRawTimestamp, + final boolean useExpensiveSaferCasting) { + return null; + } + + @Override + public Sql overwriteFinalTable(final StreamId stream, final String finalSuffix) { + return null; + } + + @Override + public Sql migrateFromV1toV2(final StreamId streamId, final String namespace, final String tableName) { + return null; + } + + @Override + public Sql prepareTablesForSoftReset(final StreamConfig stream) { + return SqlGenerator.super.prepareTablesForSoftReset(stream); + } + + @Override + public Sql clearLoadedAt(final StreamId streamId) { + return null; + } + + @Override + public boolean shouldRetry(final Exception e) { + return SqlGenerator.super.shouldRetry(e); + } +} diff --git a/airbyte-integrations/connectors/destination-clickhouse/build.gradle b/airbyte-integrations/connectors/destination-clickhouse/build.gradle index 0386841d5f45..292ed964ecdc 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse/build.gradle @@ -4,9 +4,9 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' - features = ['db-destinations'] - useLocalCdk = false + cdkVersionRequired = '0.13.3' + features = ['db-destinations', 's3-destinations', 'typing-deduping'] + useLocalCdk = true } //remove once upgrading the CDK version to 0.4.x or later diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java index 45a9d7cc8f08..fc641e3b12a2 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java @@ -16,6 +16,8 @@ import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination; import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.RawOnlySqlGenerator; import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; @@ -87,7 +89,7 @@ public AirbyteConnectionStatus check(final JsonNode config) { final JdbcDatabase database = getDatabase(dataSource); final NamingConventionTransformer namingResolver = getNamingResolver(); final String outputSchema = namingResolver.getIdentifier(config.get(JdbcUtils.DATABASE_KEY).asText()); - attemptSQLCreateAndDropTableOperations(outputSchema, database, namingResolver, getSqlOperations()); + attemptTableOperations(outputSchema, database, namingResolver, getSqlOperations(), false); return new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED); } catch (final Exception e) { LOGGER.error("Exception while checking connection: ", e); @@ -115,4 +117,8 @@ public static void main(final String[] args) throws Exception { LOGGER.info("completed destination: {}", ClickhouseDestination.class); } + @Override + protected JdbcSqlGenerator getSqlGenerator() { + return new RawOnlySqlGenerator(new ClickhouseSQLNameTransformer()); + } } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java index 76d2fa56af89..478143f0c9b3 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java @@ -10,7 +10,7 @@ import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.cdk.integrations.destination.jdbc.JdbcSqlOperations; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -60,7 +60,7 @@ public void executeTransaction(final JdbcDatabase database, final List q @Override public void insertRecordsInternal(final JdbcDatabase database, - final List records, + final List records, final String schemaName, final String tmpTableName) throws SQLException { @@ -102,4 +102,10 @@ public void insertRecordsInternal(final JdbcDatabase database, }); } + @Override + protected void insertRecordsInternalV2(final JdbcDatabase database, final List records, final String schemaName, + final String tableName) + throws Exception { + + } } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json index 4f3c51333f8a..01da3d569796 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json @@ -58,6 +58,19 @@ "type": "boolean", "default": false, "order": 6 + }, + "raw_data_schema": { + "type": "string", + "description": "The schema to write raw tables into (default: airbyte_internal)", + "title": "Raw Table Schema Name", + "order": 7 + }, + "use_v2_raw_table": { + "type": "boolean", + "description": "(For internal use.) Declares this uses the v2 raw table schema", + "title": "(Internal) Use V2 Raw Table Schema", + "airbyte_hidden": true, + "default": true } } } From 08f0173e1fb1e05dfcede946494cd4b7c4f22d1c Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Thu, 25 Jan 2024 15:15:34 -0800 Subject: [PATCH 02/96] all sorts of stuff --- .../java/airbyte-cdk/core/build.gradle | 7 ++ .../util/ConfiguredCatalogUtil.kt | 18 +++++ .../src/main/resources/version.properties | 2 +- .../jdbc/AbstractJdbcDestination.java | 76 +++++++++---------- .../typing_deduping/JdbcV1V2Migrator.java | 2 +- .../typing_deduping/RawOnlySqlGenerator.kt | 8 +- .../airbyte-cdk/typing-deduping/build.gradle | 1 + .../typing_deduping/DefaultTyperDeduper.java | 12 +-- .../typing_deduping/LoggingUtil.kt | 8 ++ .../NoMigrationNoFinalTableTyperDeduper.kt | 55 ++++++++++++++ .../typing_deduping/NoOpRawTableTDLock.kt | 27 +++++++ .../NoOpTyperDeduperWithV1V2Migrations.java | 51 +------------ .../typing_deduping/TyperDeduperUtil.kt | 17 +++++ .../destination-clickhouse/build.gradle | 2 +- .../clickhouse/ClickhouseDestination.java | 5 ++ .../clickhouse/ClickhouseSqlOperations.java | 24 +++--- 16 files changed, 198 insertions(+), 117 deletions(-) create mode 100644 airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConfiguredCatalogUtil.kt create mode 100644 airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/LoggingUtil.kt create mode 100644 airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoMigrationNoFinalTableTyperDeduper.kt create mode 100644 airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpRawTableTDLock.kt create mode 100644 airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduperUtil.kt diff --git a/airbyte-cdk/java/airbyte-cdk/core/build.gradle b/airbyte-cdk/java/airbyte-cdk/core/build.gradle index cfef9562cbbe..69c13d5fc92b 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/core/build.gradle @@ -1,3 +1,6 @@ +plugins { + id 'org.jetbrains.kotlin.jvm' version '1.9.22' +} java { compileJava { @@ -103,4 +106,8 @@ dependencies { testImplementation 'org.apache.commons:commons-lang3:3.11' testImplementation 'org.xerial.snappy:snappy-java:1.1.8.4' testImplementation 'org.mockito:mockito-core:4.6.1' + implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8" } + +compileKotlin {} +compileTestKotlin {} diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConfiguredCatalogUtil.kt b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConfiguredCatalogUtil.kt new file mode 100644 index 000000000000..77b1b431505d --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConfiguredCatalogUtil.kt @@ -0,0 +1,18 @@ +package io.airbyte.cdk.integrations.util + +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog +import org.apache.commons.lang3.StringUtils + +/** + * For streams in [catalog] which do not have a namespace specified, explicitly set their namespace + * to the [defaultNamespace] + */ + fun addDefaultNamespaceToStreams(catalog: ConfiguredAirbyteCatalog, defaultNamespace: String) { + // TODO: This logic exists in all V2 destinations. + // This is sad that if we forget to add this, there will be a null pointer during parseCatalog + for (stream in catalog.streams) { + if (StringUtils.isEmpty(stream.stream.namespace)) { + stream.stream.namespace = defaultNamespace + } + } +} diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties index e701908e301c..e32751579c90 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties @@ -1 +1 @@ -version=0.14.1 +version=0.15.0 diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java index 0741f441287f..f444d9888717 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java @@ -5,6 +5,7 @@ package io.airbyte.cdk.integrations.destination.jdbc; import static io.airbyte.cdk.integrations.base.errors.messages.ErrorMessage.getErrorMessage; +import static io.airbyte.cdk.integrations.util.ConfiguredCatalogUtilKt.addDefaultNamespaceToStreams; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; @@ -40,7 +41,6 @@ import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.sql.SQLException; import java.util.List; import java.util.Map; @@ -49,7 +49,6 @@ import java.util.function.Consumer; import javax.sql.DataSource; import org.apache.commons.lang3.NotImplementedException; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -276,45 +275,16 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN final ConfiguredAirbyteCatalog catalog, final Consumer outputRecordCollector) throws Exception { - final DataSource dataSource = getDataSource(config); - final JdbcDatabase database = getDatabase(dataSource); + final JdbcDatabase database = getDatabase(getDataSource(config)); + final String defaultNamespace; + final TyperDeduper typerDeduper; if (TypingAndDedupingFlag.isDestinationV2()) { - // TODO: This logic exists in all V2 destinations. - // This is sad that if we forget to add this, there will be a null pointer during parseCatalog - final String defaultNamespace = config.get("schema").asText(); - for (final ConfiguredAirbyteStream stream : catalog.getStreams()) { - if (StringUtils.isEmpty(stream.getStream().getNamespace())) { - stream.getStream().setNamespace(defaultNamespace); - } - } - final JdbcSqlGenerator sqlGenerator = getSqlGenerator(); - // SqlGenerator should be ok here because the catalog parser just uses "buildStreamid" - final ParsedCatalog parsedCatalog = TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE) - .map(override -> new CatalogParser(sqlGenerator, override)) - .orElse(new CatalogParser(sqlGenerator)) - .parseCatalog(catalog); - final String databaseName = getDatabaseName(config); - final var migrator = new JdbcV1V2Migrator(namingResolver, database, databaseName); - final NoopV2TableMigrator v2TableMigrator = new NoopV2TableMigrator(); - final DestinationHandler destinationHandler = getDestinationHandler(databaseName, database); - final boolean disableTypeDedupe = !config.has(DISABLE_TYPE_DEDUPE) || config.get(DISABLE_TYPE_DEDUPE).asBoolean(false); - final TyperDeduper typerDeduper; - if (disableTypeDedupe) { - typerDeduper = new NoOpTyperDeduperWithV1V2Migrations<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator, - 8); - } else { - typerDeduper = - new DefaultTyperDeduper<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator, 8); - } - return JdbcBufferedConsumerFactory.createAsync( - outputRecordCollector, - database, - sqlOperations, - namingResolver, - config, - catalog, - defaultNamespace, - typerDeduper); + defaultNamespace = config.get("schema").asText(); + addDefaultNamespaceToStreams(catalog, defaultNamespace); + typerDeduper = getV2TyperDeduper(config, catalog, database); + } else { + defaultNamespace = null; + typerDeduper = new NoopTyperDeduper(); } return JdbcBufferedConsumerFactory.createAsync( outputRecordCollector, @@ -323,8 +293,30 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN namingResolver, config, catalog, - null, - new NoopTyperDeduper()); + defaultNamespace, + typerDeduper); + } + + private TyperDeduper getV2TyperDeduper(final JsonNode config, final ConfiguredAirbyteCatalog catalog, final JdbcDatabase database) { + final JdbcSqlGenerator sqlGenerator = getSqlGenerator(); + final ParsedCatalog parsedCatalog = TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE) + .map(override -> new CatalogParser(sqlGenerator, override)) + .orElse(new CatalogParser(sqlGenerator)) + .parseCatalog(catalog); + final String databaseName = getDatabaseName(config); + final var migrator = new JdbcV1V2Migrator(namingResolver, database, databaseName); + final NoopV2TableMigrator v2TableMigrator = new NoopV2TableMigrator(); + final DestinationHandler destinationHandler = getDestinationHandler(databaseName, database); + final boolean disableTypeDedupe = !config.has(DISABLE_TYPE_DEDUPE) || config.get(DISABLE_TYPE_DEDUPE).asBoolean(false); + final TyperDeduper typerDeduper; + if (disableTypeDedupe) { + typerDeduper = new NoOpTyperDeduperWithV1V2Migrations<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator, + 8); + } else { + typerDeduper = + new DefaultTyperDeduper<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator, 8); + } + return typerDeduper; } } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcV1V2Migrator.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcV1V2Migrator.java index d5374140fc3e..1bf0c66d6b7f 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcV1V2Migrator.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcV1V2Migrator.java @@ -37,7 +37,7 @@ protected boolean doesAirbyteInternalNamespaceExist(final StreamConfig streamCon String retrievedSchema = ""; try (ResultSet columns = database.getMetaData().getSchemas(databaseName, streamConfig.id().rawNamespace())) { while (columns.next()) { - retrievedSchema = columns.getString("TABLE_SCHEM"); + retrievedSchema = columns.getString("TABLE_SCHEMA"); // Catalog can be null, so don't do anything with it. String catalog = columns.getString("TABLE_CATALOG"); } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt index 7994a22e2923..9b7f378b70a5 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt @@ -14,19 +14,19 @@ import java.util.* class RawOnlySqlGenerator(namingTransformer: NamingConventionTransformer?) : JdbcSqlGenerator(namingTransformer) { override fun getStructType(): DataType<*>? { - return null + throw NotImplementedError("This Destination Does not support typing") } override fun getArrayType(): DataType<*>? { - return null + throw NotImplementedError("This Destination Does not support typing") } override fun getWidestType(): DataType<*>? { - return null + throw NotImplementedError("This Destination Does not support typing") } override fun getDialect(): SQLDialect? { - return null + throw NotImplementedError("This Destination Does not support typing") } override fun extractRawDataFields( diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle b/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle index fc3ba8f061e8..a2bc6649f80c 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle @@ -1,5 +1,6 @@ plugins { id 'java-library' + id 'org.jetbrains.kotlin.jvm' version '1.9.22' } dependencies { diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java index d01f47060ba4..2678500a271b 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java @@ -7,15 +7,14 @@ import static io.airbyte.cdk.integrations.base.IntegrationRunner.TYPE_AND_DEDUPE_THREAD_NAME; import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.countOfTypingDedupingThreads; import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.reduceExceptions; +import static io.airbyte.integrations.base.destination.typing_deduping.TyperDeduperUtilKt.prepareAllSchemas; import static java.util.Collections.singleton; -import com.google.common.collect.Streams; import io.airbyte.cdk.integrations.destination.StreamSyncSummary; import io.airbyte.protocol.models.v0.DestinationSyncMode; import io.airbyte.protocol.models.v0.StreamDescriptor; import java.util.HashSet; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -104,14 +103,7 @@ public DefaultTyperDeduper( } private void prepareSchemas(final ParsedCatalog parsedCatalog) throws Exception { - final var rawSchema = parsedCatalog.streams().stream().map(stream -> stream.id().rawNamespace()); - final var finalSchema = parsedCatalog.streams().stream().map(stream -> stream.id().finalNamespace()); - final var createAllSchemasSql = Streams.concat(rawSchema, finalSchema) - .filter(Objects::nonNull) - .distinct() - .map(sqlGenerator::createSchema) - .toList(); - destinationHandler.execute(Sql.concat(createAllSchemasSql)); + prepareAllSchemas(parsedCatalog, sqlGenerator, destinationHandler); } @Override diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/LoggingUtil.kt b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/LoggingUtil.kt new file mode 100644 index 000000000000..ccd73acdc59f --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/LoggingUtil.kt @@ -0,0 +1,8 @@ +package io.airbyte.integrations.base.destination.typing_deduping + +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +fun R.logger(): Lazy { + return lazy { LoggerFactory.getLogger(this.javaClass) } +} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoMigrationNoFinalTableTyperDeduper.kt b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoMigrationNoFinalTableTyperDeduper.kt new file mode 100644 index 000000000000..4ad971532010 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoMigrationNoFinalTableTyperDeduper.kt @@ -0,0 +1,55 @@ +package io.airbyte.integrations.base.destination.typing_deduping + +import io.airbyte.cdk.integrations.base.IntegrationRunner +import io.airbyte.cdk.integrations.destination.StreamSyncSummary +import io.airbyte.protocol.models.v0.StreamDescriptor +import org.apache.commons.lang3.concurrent.BasicThreadFactory +import org.slf4j.Logger +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.locks.Lock + +class NoMigrationNoFinalTableTyperDeduper( + private val sqlGenerator: SqlGenerator, + private val destinationHandler: DestinationHandler, + private val parsedCatalog: ParsedCatalog, + ): TyperDeduper { + + private val executorService: ExecutorService = Executors.newFixedThreadPool( + FutureUtils.countOfTypingDedupingThreads(8), + BasicThreadFactory.Builder().namingPattern(IntegrationRunner.TYPE_AND_DEDUPE_THREAD_NAME) + .build() + ); + + private val logger: Logger by logger() + + + override fun prepareTables() { + prepareAllSchemas(parsedCatalog, sqlGenerator, destinationHandler) + } + + override fun typeAndDedupe( + originalNamespace: String?, + originalName: String?, + mustRun: Boolean + ) { + logger.info("Destination Does not support Typing and Deduping, Skipping") + } + + override fun getRawTableInsertLock(originalNamespace: String?, originalName: String?): Lock { + return NoOpRawTableTDLock(); + } + + override fun typeAndDedupe(streamSyncSummaries: MutableMap?) { + logger.info("Destination Does not support Typing and Deduping, Skipping 'typeAndDedupe'") + } + + override fun commitFinalTables() { + logger.info("Destination does not support final tables, Skipping 'commitFinalTables'") + } + + override fun cleanup() { + logger.info("Shutting down type and dedupe threads") + executorService.shutdown() + } +} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpRawTableTDLock.kt b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpRawTableTDLock.kt new file mode 100644 index 000000000000..3cc88d70fdd5 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpRawTableTDLock.kt @@ -0,0 +1,27 @@ +package io.airbyte.integrations.base.destination.typing_deduping + +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.Condition +import java.util.concurrent.locks.Lock + +class NoOpRawTableTDLock: Lock { + override fun lock() {} + + override fun lockInterruptibly() {} + + override fun tryLock(): Boolean { + // To mimic NoOp behavior always return true that lock is acquired + return true + } + + override fun tryLock(time: Long, unit: TimeUnit): Boolean { + return tryLock() + } + + override fun unlock() {} + + override fun newCondition(): Condition { + // Always throw exception to avoid callers from using this path + throw UnsupportedOperationException("This lock implementation does not support retrieving a Condition") + } +} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java index f35d1a92356d..dd33e4fb3bde 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java @@ -7,20 +7,17 @@ import static io.airbyte.cdk.integrations.base.IntegrationRunner.TYPE_AND_DEDUPE_THREAD_NAME; import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.countOfTypingDedupingThreads; import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.reduceExceptions; +import static io.airbyte.integrations.base.destination.typing_deduping.TyperDeduperUtilKt.prepareAllSchemas; -import com.google.common.collect.Streams; import io.airbyte.cdk.integrations.destination.StreamSyncSummary; import io.airbyte.protocol.models.v0.StreamDescriptor; import java.util.HashSet; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.concurrent.BasicThreadFactory; @@ -56,14 +53,7 @@ public NoOpTyperDeduperWithV1V2Migrations(final SqlGenerator stream.id().rawNamespace()); - final var finalSchema = parsedCatalog.streams().stream().map(stream -> stream.id().finalNamespace()); - final var createAllSchemasSql = Streams.concat(rawSchema, finalSchema) - .filter(Objects::nonNull) - .distinct() - .map(sqlGenerator::createSchema) - .toList(); - destinationHandler.execute(Sql.concat(createAllSchemasSql)); + prepareAllSchemas(parsedCatalog, sqlGenerator, destinationHandler); } @Override @@ -96,42 +86,7 @@ public void typeAndDedupe(final String originalNamespace, final String originalN @Override public Lock getRawTableInsertLock(final String originalNamespace, final String originalName) { - return new Lock() { - - @Override - public void lock() { - - } - - @Override - public void lockInterruptibly() { - - } - - @Override - public boolean tryLock() { - // To mimic NoOp behavior always return true that lock is acquired - return true; - } - - @Override - public boolean tryLock(final long time, final TimeUnit unit) { - // To mimic NoOp behavior always return true that lock is acquired - return true; - } - - @Override - public void unlock() { - - } - - @Override - public Condition newCondition() { - // Always throw exception to avoid callers from using this path - throw new UnsupportedOperationException("This lock implementation does not support retrieving a Condition"); - } - - }; + return new NoOpRawTableTDLock(); } @Override diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduperUtil.kt b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduperUtil.kt new file mode 100644 index 000000000000..833b63df1e77 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduperUtil.kt @@ -0,0 +1,17 @@ +package io.airbyte.integrations.base.destination.typing_deduping + +import com.google.common.collect.Streams +import java.util.* + +fun prepareAllSchemas(parsedCatalog: ParsedCatalog, sqlGenerator: SqlGenerator, destinationHandler: DestinationHandler) { + val rawSchema = + parsedCatalog.streams.stream().map { stream: StreamConfig -> stream.id.rawNamespace } + val finalSchema = + parsedCatalog.streams.stream().map { stream: StreamConfig -> stream.id.finalNamespace } + val createAllSchemasSql = Streams.concat(rawSchema, finalSchema) + .filter { obj: String? -> Objects.nonNull(obj) } + .distinct() + .map { schema: String? -> sqlGenerator.createSchema(schema) } + .toList() + destinationHandler.execute(Sql.concat(createAllSchemasSql)) +} diff --git a/airbyte-integrations/connectors/destination-clickhouse/build.gradle b/airbyte-integrations/connectors/destination-clickhouse/build.gradle index 292ed964ecdc..feb110ecd7a6 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.13.3' + cdkVersionRequired = '0.15.0' features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = true } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java index fc641e3b12a2..1cb1b37fc3f0 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java @@ -121,4 +121,9 @@ public static void main(final String[] args) throws Exception { protected JdbcSqlGenerator getSqlGenerator() { return new RawOnlySqlGenerator(new ClickhouseSQLNameTransformer()); } + + @Override + public boolean isV2Destination() { + return true; + } } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java index 478143f0c9b3..780eef6d9224 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java @@ -36,18 +36,22 @@ public boolean isSchemaRequired() { @Override public String createTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { return String.format( - "CREATE TABLE IF NOT EXISTS %s.%s ( \n" - + "%s String,\n" - + "%s String,\n" - + "%s DateTime64(3, 'GMT') DEFAULT now(),\n" - + "PRIMARY KEY(%s)\n" - + ")\n" - + "ENGINE = MergeTree;\n", + """ + CREATE TABLE IF NOT EXISTS %s.%s ( + %s String, + %s JSON, + %s DateTime64(3, 'GMT') DEFAULT now(), + %s DateTime64(3, 'GMT') DEFAULT now(), + PRIMARY KEY(%s) + ) + ENGINE = MergeTree; + """, schemaName, tableName, - JavaBaseConstants.COLUMN_NAME_AB_ID, + JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, JavaBaseConstants.COLUMN_NAME_DATA, - JavaBaseConstants.COLUMN_NAME_EMITTED_AT, - JavaBaseConstants.COLUMN_NAME_AB_ID); + JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, + JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT, + JavaBaseConstants.COLUMN_NAME_AB_RAW_ID); } @Override From d2cb631075880cc34d60a736307390305f14002a Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Fri, 26 Jan 2024 17:33:16 -0800 Subject: [PATCH 03/96] working sync --- .../integrations/base/DestinationConfig.java | 1 + .../base/ssh/SshWrappedDestination.java | 12 +++++- .../util/ConfiguredCatalogUtil.kt | 5 ++- .../airbyte-cdk/db-destinations/build.gradle | 3 ++ .../jdbc/AbstractJdbcDestination.java | 6 ++- .../typing_deduping/JdbcV1V2Migrator.java | 2 +- .../typing_deduping/RawOnlySqlGenerator.kt | 2 +- .../airbyte-cdk/typing-deduping/build.gradle | 3 ++ .../NoOpTyperDeduperWithV1V2Migrations.java | 42 +++++++++++-------- .../clickhouse/ClickhouseDestination.java | 5 +++ .../clickhouse/ClickhouseSqlOperations.java | 4 +- docs/integrations/destinations/clickhouse.md | 12 ++++++ 12 files changed, 72 insertions(+), 25 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/DestinationConfig.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/DestinationConfig.java index 300a13d6a401..03802a32ac22 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/DestinationConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/DestinationConfig.java @@ -41,6 +41,7 @@ public static void initialize(final JsonNode root, final boolean isV2Destination config = new DestinationConfig(); config.root = root; config.isV2Destination = isV2Destination; + LOGGER.info("Initializing Destination: is " + (isV2Destination ? "": "not" + " a V2 Destination" )); } else { LOGGER.warn("Singleton was already initialized."); } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshWrappedDestination.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshWrappedDestination.java index 08f4d794d92b..e732251d6dfb 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshWrappedDestination.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshWrappedDestination.java @@ -4,7 +4,13 @@ package io.airbyte.cdk.integrations.base.ssh; -import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.*; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.CONNECTION_OPTIONS_KEY; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.GLOBAL_HEARTBEAT_INTERVAL_DEFAULT_IN_MILLIS; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.GLOBAL_HEARTBEAT_INTERVAL_KEY; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.SESSION_HEARTBEAT_INTERVAL_DEFAULT_IN_MILLIS; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.SESSION_HEARTBEAT_INTERVAL_KEY; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.getInstance; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.sshWrap; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -129,4 +135,8 @@ protected SshTunnel getTunnelInstance(final JsonNode config) throws Exception { : getInstance(config, hostKey, portKey); } + @Override + public boolean isV2Destination() { + return this.delegate.isV2Destination(); + } } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConfiguredCatalogUtil.kt b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConfiguredCatalogUtil.kt index 77b1b431505d..44d8245effd6 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConfiguredCatalogUtil.kt +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConfiguredCatalogUtil.kt @@ -7,7 +7,10 @@ import org.apache.commons.lang3.StringUtils * For streams in [catalog] which do not have a namespace specified, explicitly set their namespace * to the [defaultNamespace] */ - fun addDefaultNamespaceToStreams(catalog: ConfiguredAirbyteCatalog, defaultNamespace: String) { + fun addDefaultNamespaceToStreams(catalog: ConfiguredAirbyteCatalog, defaultNamespace: String?) { + if (defaultNamespace == null) { + return + } // TODO: This logic exists in all V2 destinations. // This is sad that if we forget to add this, there will be a null pointer during parseCatalog for (stream in catalog.streams) { diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle index b83efde9dde3..d2dbd1790440 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle @@ -96,3 +96,6 @@ dependencies { testImplementation libs.junit.jupiter.system.stubs } + +compileKotlin {} +compileTestKotlin {} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java index f444d9888717..3a2c52a23af9 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java @@ -71,6 +71,10 @@ protected SqlOperations getSqlOperations() { return sqlOperations; } + protected String getConfigSchemaKey() { + return "schema"; + } + public AbstractJdbcDestination(final String driverClass, final NamingConventionTransformer namingResolver, final SqlOperations sqlOperations) { @@ -279,7 +283,7 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN final String defaultNamespace; final TyperDeduper typerDeduper; if (TypingAndDedupingFlag.isDestinationV2()) { - defaultNamespace = config.get("schema").asText(); + defaultNamespace = config.get(getConfigSchemaKey()).asText(); addDefaultNamespaceToStreams(catalog, defaultNamespace); typerDeduper = getV2TyperDeduper(config, catalog, database); } else { diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcV1V2Migrator.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcV1V2Migrator.java index 1bf0c66d6b7f..d5374140fc3e 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcV1V2Migrator.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcV1V2Migrator.java @@ -37,7 +37,7 @@ protected boolean doesAirbyteInternalNamespaceExist(final StreamConfig streamCon String retrievedSchema = ""; try (ResultSet columns = database.getMetaData().getSchemas(databaseName, streamConfig.id().rawNamespace())) { while (columns.next()) { - retrievedSchema = columns.getString("TABLE_SCHEMA"); + retrievedSchema = columns.getString("TABLE_SCHEM"); // Catalog can be null, so don't do anything with it. String catalog = columns.getString("TABLE_CATALOG"); } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt index 9b7f378b70a5..834e9d409bef 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt @@ -11,7 +11,7 @@ import org.jooq.Field import org.jooq.SQLDialect import java.util.* -class RawOnlySqlGenerator(namingTransformer: NamingConventionTransformer?) : +class RawOnlySqlGenerator(private val namingTransformer: NamingConventionTransformer) : JdbcSqlGenerator(namingTransformer) { override fun getStructType(): DataType<*>? { throw NotImplementedError("This Destination Does not support typing") diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle b/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle index a2bc6649f80c..4160813789d6 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle @@ -25,3 +25,6 @@ java { options.compilerArgs.remove("-Werror") } } + +compileKotlin {} +compileTestKotlin {} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java index dd33e4fb3bde..3bbc53fae3b5 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java @@ -19,7 +19,9 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.locks.Lock; +import kotlin.NotImplementedError; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.concurrent.BasicThreadFactory; /** @@ -58,25 +60,29 @@ private void prepareSchemas(final ParsedCatalog parsedCatalog) throws Exception @Override public void prepareTables() throws Exception { - log.info("ensuring schemas exist for prepareTables with V1V2 migrations"); - prepareSchemas(parsedCatalog); - final Set>> prepareTablesTasks = new HashSet<>(); - for (final StreamConfig stream : parsedCatalog.streams()) { - prepareTablesTasks.add(CompletableFuture.supplyAsync(() -> { - // Migrate the Raw Tables if this is the first v2 sync after a v1 sync - try { - log.info("Migrating V1->V2 for stream {}", stream.id()); - v1V2Migrator.migrateIfNecessary(sqlGenerator, destinationHandler, stream); - log.info("Migrating V2 legacy for stream {}", stream.id()); - v2TableMigrator.migrateIfNecessary(stream); - return Optional.empty(); - } catch (final Exception e) { - return Optional.of(e); - } - }, executorService)); + try { + log.info("ensuring schemas exist for prepareTables with V1V2 migrations"); + prepareSchemas(parsedCatalog); + final Set>> prepareTablesTasks = new HashSet<>(); + for (final StreamConfig stream : parsedCatalog.streams()) { + prepareTablesTasks.add(CompletableFuture.supplyAsync(() -> { + // Migrate the Raw Tables if this is the first v2 sync after a v1 sync + try { + log.info("Migrating V1->V2 for stream {}", stream.id()); + v1V2Migrator.migrateIfNecessary(sqlGenerator, destinationHandler, stream); + log.info("Migrating V2 legacy for stream {}", stream.id()); + v2TableMigrator.migrateIfNecessary(stream); + return Optional.empty(); + } catch (final Exception e) { + return Optional.of(e); + } + }, executorService)); + } + CompletableFuture.allOf(prepareTablesTasks.toArray(CompletableFuture[]::new)).join(); + reduceExceptions(prepareTablesTasks, "The following exceptions were thrown attempting to prepare tables:\n"); + } catch (NotImplementedError | NotImplementedException e) { + log.warn("Could not prepare schemas or tables because components are not implemented, this should not be required for this destination to succeed"); } - CompletableFuture.allOf(prepareTablesTasks.toArray(CompletableFuture[]::new)).join(); - reduceExceptions(prepareTablesTasks, "The following exceptions were thrown attempting to prepare tables:\n"); } @Override diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java index 1cb1b37fc3f0..ab53950f686d 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java @@ -126,4 +126,9 @@ protected JdbcSqlGenerator getSqlGenerator() { public boolean isV2Destination() { return true; } + + @Override + protected String getConfigSchemaKey() { + return "database"; + } } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java index 780eef6d9224..6ff7f462a8ef 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java @@ -39,7 +39,7 @@ public String createTableQuery(final JdbcDatabase database, final String schemaN """ CREATE TABLE IF NOT EXISTS %s.%s ( %s String, - %s JSON, + %s String, %s DateTime64(3, 'GMT') DEFAULT now(), %s DateTime64(3, 'GMT') DEFAULT now(), PRIMARY KEY(%s) @@ -110,6 +110,6 @@ public void insertRecordsInternal(final JdbcDatabase database, protected void insertRecordsInternalV2(final JdbcDatabase database, final List records, final String schemaName, final String tableName) throws Exception { - + insertRecordsInternal(database, records, schemaName, tableName); } } diff --git a/docs/integrations/destinations/clickhouse.md b/docs/integrations/destinations/clickhouse.md index 02446ba825f6..eccadec29802 100644 --- a/docs/integrations/destinations/clickhouse.md +++ b/docs/integrations/destinations/clickhouse.md @@ -44,6 +44,18 @@ You can create such a user by running: ``` GRANT CREATE ON * TO airbyte_user; +GRANT CREATE ON default * TO airbyte_user + +GRANT DROP ON * TO airbyte_user +GRANT TRUNCATE ON * TO airbyte_user +GRANT INSERT ON * TO airbyte_user +GRANT SELECT ON * TO airbyte_user +GRANT CREATE DATABASE ON airbyte_internal.* TO airbyte_user +GRANT CREATE TABLE ON airbyte_internal.* TO airbyte_user +GRANT DROP ON airbyte_internal.* TO airbyte_user +GRANT TRUNCATE ON airbyte_internal.* TO airbyte_user +GRANT INSERT ON airbyte_internal.* TO airbyte_user +GRANT SELECT ON airbyte_internal.* TO airbyte_user ``` You can also use a pre-existing user but we highly recommend creating a dedicated user for Airbyte. From 35c349318c6f90994fea400106511b2e38e056a1 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Mon, 29 Jan 2024 15:06:59 -0800 Subject: [PATCH 04/96] add breaking change metadata --- .../destination-clickhouse/metadata.yaml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml b/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml index cf10ca7aa667..00065e12efe0 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml +++ b/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml @@ -2,22 +2,27 @@ data: connectorSubtype: database connectorType: destination definitionId: ce0d828e-1dc4-496c-b122-2da42e637e48 - dockerImageTag: 0.2.5 + dockerImageTag: 1.0.0 dockerRepository: airbyte/destination-clickhouse githubIssueLabel: destination-clickhouse icon: clickhouse.svg license: MIT name: Clickhouse - normalizationConfig: - normalizationIntegrationType: clickhouse - normalizationRepository: airbyte/normalization-clickhouse - normalizationTag: 0.4.3 registries: cloud: dockerRepository: airbyte/destination-clickhouse-strict-encrypt enabled: true oss: enabled: true + releases: + breakingChanges: + 1.0.0: + message: > + This version removes the option to use "normalization" with clickhouse. It also changes + the schema and database of Airbyte's "raw" tables to be compatible with the new + [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2) + format. These changes will likely require updates to downstream dbt / SQL models. + Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/clickhouse supportsDbt: false From 8004b508530f8fa58127a5c15a0994e3cf69c27c Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Mon, 29 Jan 2024 15:12:11 -0800 Subject: [PATCH 05/96] formatting --- .../integrations/base/DestinationConfig.java | 2 +- .../base/ssh/SshWrappedDestination.java | 1 + .../jdbc/AbstractJdbcDestination.java | 8 +++---- .../typing_deduping/NoOpSqlGenerator.java | 9 +++++++- .../NoOpTyperDeduperWithV1V2Migrations.java | 3 ++- .../destination-clickhouse/metadata.yaml | 2 +- .../clickhouse/ClickhouseDestination.java | 1 + .../clickhouse/ClickhouseSqlOperations.java | 23 +++++++++++-------- 8 files changed, 31 insertions(+), 18 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/DestinationConfig.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/DestinationConfig.java index ebfde0df4612..9a7288ee4638 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/DestinationConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/DestinationConfig.java @@ -39,7 +39,7 @@ public static void initialize(final JsonNode root, final boolean isV2Destination config = new DestinationConfig(); config.root = root; config.isV2Destination = isV2Destination; - LOGGER.info("Initializing Destination: is " + (isV2Destination ? "": "not" + " a V2 Destination" )); + LOGGER.info("Initializing Destination: is " + (isV2Destination ? "" : "not" + " a V2 Destination")); } else { LOGGER.warn("Singleton was already initialized."); } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshWrappedDestination.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshWrappedDestination.java index e732251d6dfb..8513e2cda29a 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshWrappedDestination.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshWrappedDestination.java @@ -139,4 +139,5 @@ protected SshTunnel getTunnelInstance(final JsonNode config) throws Exception { public boolean isV2Destination() { return this.delegate.isV2Destination(); } + } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java index 3a2c52a23af9..1032eb242ad5 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java @@ -304,9 +304,9 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN private TyperDeduper getV2TyperDeduper(final JsonNode config, final ConfiguredAirbyteCatalog catalog, final JdbcDatabase database) { final JdbcSqlGenerator sqlGenerator = getSqlGenerator(); final ParsedCatalog parsedCatalog = TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE) - .map(override -> new CatalogParser(sqlGenerator, override)) - .orElse(new CatalogParser(sqlGenerator)) - .parseCatalog(catalog); + .map(override -> new CatalogParser(sqlGenerator, override)) + .orElse(new CatalogParser(sqlGenerator)) + .parseCatalog(catalog); final String databaseName = getDatabaseName(config); final var migrator = new JdbcV1V2Migrator(namingResolver, database, databaseName); final NoopV2TableMigrator v2TableMigrator = new NoopV2TableMigrator(); @@ -315,7 +315,7 @@ private TyperDeduper getV2TyperDeduper(final JsonNode config, final ConfiguredAi final TyperDeduper typerDeduper; if (disableTypeDedupe) { typerDeduper = new NoOpTyperDeduperWithV1V2Migrations<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator, - 8); + 8); } else { typerDeduper = new DefaultTyperDeduper<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator, 8); diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpSqlGenerator.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpSqlGenerator.java index 6a423fcb018c..fab0f4b95237 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpSqlGenerator.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpSqlGenerator.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.base.destination.typing_deduping; import java.util.Optional; @@ -37,7 +41,9 @@ public boolean existingSchemaMatchesStreamConfig(final StreamConfig stream, fina } @Override - public Sql updateTable(final StreamConfig stream, final String finalSuffix, final Optional minRawTimestamp, + public Sql updateTable(final StreamConfig stream, + final String finalSuffix, + final Optional minRawTimestamp, final boolean useExpensiveSaferCasting) { return null; } @@ -66,4 +72,5 @@ public Sql clearLoadedAt(final StreamId streamId) { public boolean shouldRetry(final Exception e) { return SqlGenerator.super.shouldRetry(e); } + } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java index 3bbc53fae3b5..a70daaef62cc 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java @@ -81,7 +81,8 @@ public void prepareTables() throws Exception { CompletableFuture.allOf(prepareTablesTasks.toArray(CompletableFuture[]::new)).join(); reduceExceptions(prepareTablesTasks, "The following exceptions were thrown attempting to prepare tables:\n"); } catch (NotImplementedError | NotImplementedException e) { - log.warn("Could not prepare schemas or tables because components are not implemented, this should not be required for this destination to succeed"); + log.warn( + "Could not prepare schemas or tables because components are not implemented, this should not be required for this destination to succeed"); } } diff --git a/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml b/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml index 00065e12efe0..e0775a1f3351 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml +++ b/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml @@ -22,7 +22,7 @@ data: the schema and database of Airbyte's "raw" tables to be compatible with the new [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2) format. These changes will likely require updates to downstream dbt / SQL models. - Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. + Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/clickhouse supportsDbt: false diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java index ab53950f686d..77b3347944ca 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java @@ -131,4 +131,5 @@ public boolean isV2Destination() { protected String getConfigSchemaKey() { return "database"; } + } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java index 6ff7f462a8ef..66c7dcf023c0 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java @@ -37,15 +37,15 @@ public boolean isSchemaRequired() { public String createTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { return String.format( """ - CREATE TABLE IF NOT EXISTS %s.%s ( - %s String, - %s String, - %s DateTime64(3, 'GMT') DEFAULT now(), - %s DateTime64(3, 'GMT') DEFAULT now(), - PRIMARY KEY(%s) - ) - ENGINE = MergeTree; - """, + CREATE TABLE IF NOT EXISTS %s.%s ( + %s String, + %s String, + %s DateTime64(3, 'GMT') DEFAULT now(), + %s DateTime64(3, 'GMT') DEFAULT now(), + PRIMARY KEY(%s) + ) + ENGINE = MergeTree; + """, schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, JavaBaseConstants.COLUMN_NAME_DATA, @@ -107,9 +107,12 @@ public void insertRecordsInternal(final JdbcDatabase database, } @Override - protected void insertRecordsInternalV2(final JdbcDatabase database, final List records, final String schemaName, + protected void insertRecordsInternalV2(final JdbcDatabase database, + final List records, + final String schemaName, final String tableName) throws Exception { insertRecordsInternal(database, records, schemaName, tableName); } + } From ecaf881adc35555d90eb5f4099e9d2376f10fc40 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Mon, 29 Jan 2024 15:51:33 -0800 Subject: [PATCH 06/96] pr self review --- .../integrations/base/DestinationConfig.java | 1 - .../base/TypingAndDedupingFlag.java | 4 - .../jdbc/AbstractJdbcDestination.java | 7 ++ .../typing_deduping/RawOnlySqlGenerator.kt | 18 ++--- .../typing_deduping/LoggingUtil.kt | 3 + .../NoMigrationNoFinalTableTyperDeduper.kt | 55 -------------- .../typing_deduping/NoOpSqlGenerator.java | 76 ------------------- .../NoOpTyperDeduperWithV1V2Migrations.java | 7 +- .../typing_deduping/TyperDeduperUtil.kt | 4 + .../destination-clickhouse/build.gradle | 2 +- .../clickhouse/ClickhouseSqlOperations.java | 2 +- .../src/main/resources/spec.json | 7 -- docs/integrations/destinations/clickhouse.md | 23 +++--- 13 files changed, 38 insertions(+), 171 deletions(-) delete mode 100644 airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoMigrationNoFinalTableTyperDeduper.kt delete mode 100644 airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpSqlGenerator.java diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/DestinationConfig.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/DestinationConfig.java index 9a7288ee4638..b2f922125335 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/DestinationConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/DestinationConfig.java @@ -39,7 +39,6 @@ public static void initialize(final JsonNode root, final boolean isV2Destination config = new DestinationConfig(); config.root = root; config.isV2Destination = isV2Destination; - LOGGER.info("Initializing Destination: is " + (isV2Destination ? "" : "not" + " a V2 Destination")); } else { LOGGER.warn("Singleton was already initialized."); } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/TypingAndDedupingFlag.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/TypingAndDedupingFlag.java index 5a3e257ae429..8820b1d7017f 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/TypingAndDedupingFlag.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/TypingAndDedupingFlag.java @@ -22,8 +22,4 @@ public static Optional getRawNamespaceOverride(final String option) { } } - public static boolean isUsingV2RawTable() { - return isDestinationV2() || DestinationConfig.getInstance().getBooleanValue("use_v2_raw_table"); - } - } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java index 1032eb242ad5..763277771ff5 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java @@ -301,6 +301,13 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN typerDeduper); } + /** + * Creates the appropriate TyperDeduper class for the jdbc destination and the user's configuration + * @param config the configuration for the connection + * @param catalog the catalog for the connection + * @param database a database instance + * @return the appropriate TyperDeduper instance for this connection. + */ private TyperDeduper getV2TyperDeduper(final JsonNode config, final ConfiguredAirbyteCatalog catalog, final JdbcDatabase database) { final JdbcSqlGenerator sqlGenerator = getSqlGenerator(); final ParsedCatalog parsedCatalog = TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE) diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt index 834e9d409bef..7a8eb52499e6 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt @@ -14,47 +14,47 @@ import java.util.* class RawOnlySqlGenerator(private val namingTransformer: NamingConventionTransformer) : JdbcSqlGenerator(namingTransformer) { override fun getStructType(): DataType<*>? { - throw NotImplementedError("This Destination Does not support typing") + throw NotImplementedError("This Destination Does not support final tables") } override fun getArrayType(): DataType<*>? { - throw NotImplementedError("This Destination Does not support typing") + throw NotImplementedError("This Destination Does not support final tables") } override fun getWidestType(): DataType<*>? { - throw NotImplementedError("This Destination Does not support typing") + throw NotImplementedError("This Destination Does not support final tables") } override fun getDialect(): SQLDialect? { - throw NotImplementedError("This Destination Does not support typing") + throw NotImplementedError("This Destination Does not support final tables") } override fun extractRawDataFields( columns: LinkedHashMap, useExpensiveSaferCasting: Boolean ): List>? { - return null + throw NotImplementedError("This Destination Does not support final tables") } override fun buildAirbyteMetaColumn(columns: LinkedHashMap): Field<*>? { - return null + throw NotImplementedError("This Destination Does not support final tables") } override fun cdcDeletedAtNotNullCondition(): Condition? { - return null + throw NotImplementedError("This Destination Does not support final tables") } override fun getRowNumber( primaryKey: List, cursorField: Optional ): Field? { - return null + throw NotImplementedError("This Destination Does not support final tables") } override fun existingSchemaMatchesStreamConfig( stream: StreamConfig, existingTable: TableDefinition ): Boolean { - return false + throw NotImplementedError("This Destination Does not support final tables") } } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/LoggingUtil.kt b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/LoggingUtil.kt index ccd73acdc59f..ed3552b0ff70 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/LoggingUtil.kt +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/LoggingUtil.kt @@ -3,6 +3,9 @@ package io.airbyte.integrations.base.destination.typing_deduping import org.slf4j.Logger import org.slf4j.LoggerFactory +/** + * Extends all classes to lazily initialize a Slf4j [Logger] + */ fun R.logger(): Lazy { return lazy { LoggerFactory.getLogger(this.javaClass) } } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoMigrationNoFinalTableTyperDeduper.kt b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoMigrationNoFinalTableTyperDeduper.kt deleted file mode 100644 index 4ad971532010..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoMigrationNoFinalTableTyperDeduper.kt +++ /dev/null @@ -1,55 +0,0 @@ -package io.airbyte.integrations.base.destination.typing_deduping - -import io.airbyte.cdk.integrations.base.IntegrationRunner -import io.airbyte.cdk.integrations.destination.StreamSyncSummary -import io.airbyte.protocol.models.v0.StreamDescriptor -import org.apache.commons.lang3.concurrent.BasicThreadFactory -import org.slf4j.Logger -import java.util.concurrent.ExecutorService -import java.util.concurrent.Executors -import java.util.concurrent.locks.Lock - -class NoMigrationNoFinalTableTyperDeduper( - private val sqlGenerator: SqlGenerator, - private val destinationHandler: DestinationHandler, - private val parsedCatalog: ParsedCatalog, - ): TyperDeduper { - - private val executorService: ExecutorService = Executors.newFixedThreadPool( - FutureUtils.countOfTypingDedupingThreads(8), - BasicThreadFactory.Builder().namingPattern(IntegrationRunner.TYPE_AND_DEDUPE_THREAD_NAME) - .build() - ); - - private val logger: Logger by logger() - - - override fun prepareTables() { - prepareAllSchemas(parsedCatalog, sqlGenerator, destinationHandler) - } - - override fun typeAndDedupe( - originalNamespace: String?, - originalName: String?, - mustRun: Boolean - ) { - logger.info("Destination Does not support Typing and Deduping, Skipping") - } - - override fun getRawTableInsertLock(originalNamespace: String?, originalName: String?): Lock { - return NoOpRawTableTDLock(); - } - - override fun typeAndDedupe(streamSyncSummaries: MutableMap?) { - logger.info("Destination Does not support Typing and Deduping, Skipping 'typeAndDedupe'") - } - - override fun commitFinalTables() { - logger.info("Destination does not support final tables, Skipping 'commitFinalTables'") - } - - override fun cleanup() { - logger.info("Shutting down type and dedupe threads") - executorService.shutdown() - } -} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpSqlGenerator.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpSqlGenerator.java deleted file mode 100644 index fab0f4b95237..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpSqlGenerator.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.base.destination.typing_deduping; - -import java.util.Optional; -import lombok.NoArgsConstructor; - -@NoArgsConstructor -public class NoOpSqlGenerator implements SqlGenerator { - - @Override - public StreamId buildStreamId(final String namespace, final String name, final String rawNamespaceOverride) { - return null; - } - - @Override - public ColumnId buildColumnId(final String name) { - return SqlGenerator.super.buildColumnId(name); - } - - @Override - public ColumnId buildColumnId(final String name, final String suffix) { - return null; - } - - @Override - public Sql createTable(final StreamConfig stream, final String suffix, final boolean force) { - return null; - } - - @Override - public Sql createSchema(final String schema) { - return null; - } - - @Override - public boolean existingSchemaMatchesStreamConfig(final StreamConfig stream, final Object existingTable) { - return false; - } - - @Override - public Sql updateTable(final StreamConfig stream, - final String finalSuffix, - final Optional minRawTimestamp, - final boolean useExpensiveSaferCasting) { - return null; - } - - @Override - public Sql overwriteFinalTable(final StreamId stream, final String finalSuffix) { - return null; - } - - @Override - public Sql migrateFromV1toV2(final StreamId streamId, final String namespace, final String tableName) { - return null; - } - - @Override - public Sql prepareTablesForSoftReset(final StreamConfig stream) { - return SqlGenerator.super.prepareTablesForSoftReset(stream); - } - - @Override - public Sql clearLoadedAt(final StreamId streamId) { - return null; - } - - @Override - public boolean shouldRetry(final Exception e) { - return SqlGenerator.super.shouldRetry(e); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java index a70daaef62cc..6e37951202ed 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java @@ -54,15 +54,12 @@ public NoOpTyperDeduperWithV1V2Migrations(final SqlGenerator>> prepareTablesTasks = new HashSet<>(); for (final StreamConfig stream : parsedCatalog.streams()) { prepareTablesTasks.add(CompletableFuture.supplyAsync(() -> { @@ -82,7 +79,7 @@ public void prepareTables() throws Exception { reduceExceptions(prepareTablesTasks, "The following exceptions were thrown attempting to prepare tables:\n"); } catch (NotImplementedError | NotImplementedException e) { log.warn( - "Could not prepare schemas or tables because components are not implemented, this should not be required for this destination to succeed"); + "Could not prepare schemas or tables because this is not implemented for this destination, this should not be required for this destination to succeed"); } } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduperUtil.kt b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduperUtil.kt index 833b63df1e77..f1ed3d1ba38a 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduperUtil.kt +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduperUtil.kt @@ -3,6 +3,10 @@ package io.airbyte.integrations.base.destination.typing_deduping import com.google.common.collect.Streams import java.util.* +/** + * Extracts all the "raw" and "final" schemas identified in the [parsedCatalog] and ensures they + * exist in the Destination Database. + */ fun prepareAllSchemas(parsedCatalog: ParsedCatalog, sqlGenerator: SqlGenerator, destinationHandler: DestinationHandler) { val rawSchema = parsedCatalog.streams.stream().map { stream: StreamConfig -> stream.id.rawNamespace } diff --git a/airbyte-integrations/connectors/destination-clickhouse/build.gradle b/airbyte-integrations/connectors/destination-clickhouse/build.gradle index feb110ecd7a6..daf34f9f933a 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.15.0' + cdkVersionRequired = '0.17.0' features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = true } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java index 66c7dcf023c0..80f13c47c7a2 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java @@ -41,7 +41,7 @@ public String createTableQuery(final JdbcDatabase database, final String schemaN %s String, %s String, %s DateTime64(3, 'GMT') DEFAULT now(), - %s DateTime64(3, 'GMT') DEFAULT now(), + %s DateTime64(3, 'GMT') NULL, PRIMARY KEY(%s) ) ENGINE = MergeTree; diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json index 01da3d569796..26b6fb701381 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json @@ -64,13 +64,6 @@ "description": "The schema to write raw tables into (default: airbyte_internal)", "title": "Raw Table Schema Name", "order": 7 - }, - "use_v2_raw_table": { - "type": "boolean", - "description": "(For internal use.) Declares this uses the v2 raw table schema", - "title": "(Internal) Use V2 Raw Table Schema", - "airbyte_hidden": true, - "default": true } } } diff --git a/docs/integrations/destinations/clickhouse.md b/docs/integrations/destinations/clickhouse.md index eccadec29802..64c3da36f6b8 100644 --- a/docs/integrations/destinations/clickhouse.md +++ b/docs/integrations/destinations/clickhouse.md @@ -44,18 +44,17 @@ You can create such a user by running: ``` GRANT CREATE ON * TO airbyte_user; -GRANT CREATE ON default * TO airbyte_user - -GRANT DROP ON * TO airbyte_user -GRANT TRUNCATE ON * TO airbyte_user -GRANT INSERT ON * TO airbyte_user -GRANT SELECT ON * TO airbyte_user -GRANT CREATE DATABASE ON airbyte_internal.* TO airbyte_user -GRANT CREATE TABLE ON airbyte_internal.* TO airbyte_user -GRANT DROP ON airbyte_internal.* TO airbyte_user -GRANT TRUNCATE ON airbyte_internal.* TO airbyte_user -GRANT INSERT ON airbyte_internal.* TO airbyte_user -GRANT SELECT ON airbyte_internal.* TO airbyte_user +GRANT CREATE ON default * TO airbyte_user; +GRANT DROP ON * TO airbyte_user; +GRANT TRUNCATE ON * TO airbyte_user; +GRANT INSERT ON * TO airbyte_user; +GRANT SELECT ON * TO airbyte_user; +GRANT CREATE DATABASE ON airbyte_internal.* TO airbyte_user; +GRANT CREATE TABLE ON airbyte_internal.* TO airbyte_user; +GRANT DROP ON airbyte_internal.* TO airbyte_user; +GRANT TRUNCATE ON airbyte_internal.* TO airbyte_user; +GRANT INSERT ON airbyte_internal.* TO airbyte_user; +GRANT SELECT ON airbyte_internal.* TO airbyte_user; ``` You can also use a pre-existing user but we highly recommend creating a dedicated user for Airbyte. From e5cfdf4913a5f8b1cc224a3d39ac905d708e20fc Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Mon, 29 Jan 2024 15:57:33 -0800 Subject: [PATCH 07/96] formatting --- .../integrations/destination/jdbc/AbstractJdbcDestination.java | 1 + .../typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java index 763277771ff5..89b837408e55 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java @@ -303,6 +303,7 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN /** * Creates the appropriate TyperDeduper class for the jdbc destination and the user's configuration + * * @param config the configuration for the connection * @param catalog the catalog for the connection * @param database a database instance diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java index 6e37951202ed..703b8446711a 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java @@ -54,7 +54,6 @@ public NoOpTyperDeduperWithV1V2Migrations(final SqlGenerator Date: Mon, 29 Jan 2024 22:07:21 -0800 Subject: [PATCH 08/96] AirbyteLib: Installation improvements and improved error handling (#34572) --- airbyte-lib/airbyte_lib/_executor.py | 310 +++++++++++++----- .../_factories/connector_factories.py | 76 +++-- airbyte-lib/airbyte_lib/exceptions.py | 12 +- airbyte-lib/airbyte_lib/registry.py | 73 +++-- airbyte-lib/airbyte_lib/source.py | 81 ++++- airbyte-lib/airbyte_lib/validate.py | 10 +- airbyte-lib/docs/generated/airbyte_lib.html | 46 ++- .../source-test/source_test/__init__.py | 0 .../tests/integration_tests/test_install.py | 23 ++ .../integration_tests/test_integration.py | 167 +++++++--- 10 files changed, 609 insertions(+), 189 deletions(-) create mode 100644 airbyte-lib/tests/integration_tests/fixtures/source-test/source_test/__init__.py create mode 100644 airbyte-lib/tests/integration_tests/test_install.py diff --git a/airbyte-lib/airbyte_lib/_executor.py b/airbyte-lib/airbyte_lib/_executor.py index 20899f892006..a43d56249163 100644 --- a/airbyte-lib/airbyte_lib/_executor.py +++ b/airbyte-lib/airbyte_lib/_executor.py @@ -1,22 +1,23 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. from __future__ import annotations +import shlex import subprocess import sys from abc import ABC, abstractmethod -from contextlib import contextmanager +from contextlib import contextmanager, suppress from pathlib import Path -from typing import IO, TYPE_CHECKING, Any, NoReturn +from shutil import rmtree +from typing import IO, TYPE_CHECKING, Any, NoReturn, cast from airbyte_lib import exceptions as exc +from airbyte_lib.registry import ConnectorMetadata from airbyte_lib.telemetry import SourceTelemetryInfo, SourceType if TYPE_CHECKING: from collections.abc import Generator, Iterable, Iterator - from airbyte_lib.registry import ConnectorMetadata - _LATEST_VERSION = "latest" @@ -24,22 +25,37 @@ class Executor(ABC): def __init__( self, - metadata: ConnectorMetadata, + *, + name: str | None = None, + metadata: ConnectorMetadata | None = None, target_version: str | None = None, ) -> None: - self.metadata = metadata - self.enforce_version = target_version is not None - if target_version is None or target_version == _LATEST_VERSION: - self.target_version = metadata.latest_available_version - else: - self.target_version = target_version + """Initialize a connector executor. + + The 'name' param is required if 'metadata' is None. + """ + if not name and not metadata: + raise exc.AirbyteLibInternalError(message="Either name or metadata must be provided.") + + self.name: str = name or cast(ConnectorMetadata, metadata).name # metadata is not None here + self.metadata: ConnectorMetadata | None = metadata + self.enforce_version: bool = target_version is not None + + self.reported_version: str | None = None + self.target_version: str | None = None + if target_version: + if metadata and target_version == _LATEST_VERSION: + self.target_version = metadata.latest_available_version + else: + self.target_version = target_version @abstractmethod def execute(self, args: list[str]) -> Iterator[str]: pass @abstractmethod - def ensure_installation(self) -> None: + def ensure_installation(self, *, auto_fix: bool = True) -> None: + _ = auto_fix pass @abstractmethod @@ -101,71 +117,150 @@ def _stream_from_file(file: IO[str]) -> Generator[str, Any, None]: # If the exit code is not 0 or -15 (SIGTERM), raise an exception if exit_code not in (0, -15): - raise exc.AirbyteSubprocessFailedError(exit_code=exit_code) + raise exc.AirbyteSubprocessFailedError( + run_args=args, + exit_code=exit_code, + ) class VenvExecutor(Executor): def __init__( self, - metadata: ConnectorMetadata, + name: str | None = None, + *, + metadata: ConnectorMetadata | None = None, target_version: str | None = None, pip_url: str | None = None, - *, - install_if_missing: bool = False, + install_root: Path | None = None, ) -> None: - super().__init__(metadata, target_version) - self.install_if_missing = install_if_missing + """Initialize a connector executor that runs a connector in a virtual environment. + + Args: + name: The name of the connector. + metadata: (Optional.) The metadata of the connector. + target_version: (Optional.) The version of the connector to install. + pip_url: (Optional.) The pip URL of the connector to install. + install_root: (Optional.) The root directory where the virtual environment will be + created. If not provided, the current working directory will be used. + """ + super().__init__(name=name, metadata=metadata, target_version=target_version) # This is a temporary install path that will be replaced with a proper package # name once they are published. - # TODO: Replace with `f"airbyte-{self.metadata.name}"` - self.pip_url = pip_url or f"../airbyte-integrations/connectors/{self.metadata.name}" + # TODO: Replace with `f"airbyte-{self.name}"` + self.pip_url = pip_url or f"../airbyte-integrations/connectors/{self.name}" + self.install_root = install_root or Path.cwd() def _get_venv_name(self) -> str: - return f".venv-{self.metadata.name}" + return f".venv-{self.name}" + + def _get_venv_path(self) -> Path: + return self.install_root / self._get_venv_name() def _get_connector_path(self) -> Path: - return Path(self._get_venv_name(), "bin", self.metadata.name) + return self._get_venv_path() / "bin" / self.name def _run_subprocess_and_raise_on_failure(self, args: list[str]) -> None: - result = subprocess.run(args, check=False) + result = subprocess.run( + args, + check=False, + stderr=subprocess.PIPE, + ) if result.returncode != 0: - raise exc.AirbyteConnectorInstallationError from exc.AirbyteSubprocessFailedError( - exit_code=result.returncode + raise exc.AirbyteSubprocessFailedError( + run_args=args, + exit_code=result.returncode, + log_text=result.stderr.decode("utf-8"), ) def uninstall(self) -> None: - venv_name = self._get_venv_name() - if Path(venv_name).exists(): - self._run_subprocess_and_raise_on_failure(["rm", "-rf", venv_name]) + if self._get_venv_path().exists(): + rmtree(str(self._get_venv_path())) + + self.reported_version = None # Reset the reported version from the previous installation def install(self) -> None: - venv_name = self._get_venv_name() - self._run_subprocess_and_raise_on_failure([sys.executable, "-m", "venv", venv_name]) + """Install the connector in a virtual environment. + + After installation, the installed version will be stored in self.reported_version. + """ + self._run_subprocess_and_raise_on_failure( + [sys.executable, "-m", "venv", str(self._get_venv_path())] + ) - pip_path = str(Path(venv_name) / "bin" / "pip") + pip_path = str(self._get_venv_path() / "bin" / "pip") - self._run_subprocess_and_raise_on_failure([pip_path, "install", "-e", self.pip_url]) + try: + self._run_subprocess_and_raise_on_failure( + args=[pip_path, "install", *shlex.split(self.pip_url)] + ) + except exc.AirbyteSubprocessFailedError as ex: + # If the installation failed, remove the virtual environment + # Otherwise, the connector will be considered as installed and the user may not be able + # to retry the installation. + with suppress(exc.AirbyteSubprocessFailedError): + self.uninstall() - def _get_installed_version(self) -> str: + raise exc.AirbyteConnectorInstallationError from ex + + # Assuming the installation succeeded, store the installed version + self.reported_version = self._get_installed_version(raise_on_error=False, recheck=True) + + def _get_installed_version( + self, + *, + raise_on_error: bool = False, + recheck: bool = False, + ) -> str | None: """Detect the version of the connector installed. + Returns the version string if it can be detected, otherwise None. + + If raise_on_error is True, raise an exception if the version cannot be detected. + + If recheck if False and the version has already been detected, return the cached value. + In the venv, we run the following: > python -c "from importlib.metadata import version; print(version(''))" """ - venv_name = self._get_venv_name() - connector_name = self.metadata.name - return subprocess.check_output( - [ - Path(venv_name) / "bin" / "python", - "-c", - f"from importlib.metadata import version; print(version('{connector_name}'))", - ], - universal_newlines=True, - ).strip() + if not recheck and self.reported_version: + return self.reported_version + + connector_name = self.name + if not self.interpreter_path.exists(): + # No point in trying to detect the version if the interpreter does not exist + if raise_on_error: + raise exc.AirbyteLibInternalError( + message="Connector's virtual environment interpreter could not be found.", + context={ + "interpreter_path": self.interpreter_path, + }, + ) + return None + + try: + return subprocess.check_output( + [ + self.interpreter_path, + "-c", + f"from importlib.metadata import version; print(version('{connector_name}'))", + ], + universal_newlines=True, + ).strip() + except Exception: + if raise_on_error: + raise + + return None + + @property + def interpreter_path(self) -> Path: + return self._get_venv_path() / "bin" / "python" def ensure_installation( self, + *, + auto_fix: bool = True, ) -> None: """Ensure that the connector is installed in a virtual environment. @@ -176,48 +271,77 @@ def ensure_installation( Note: Version verification is not supported for connectors installed from a local path. """ - venv_name = f".venv-{self.metadata.name}" - venv_path = Path(venv_name) - if not venv_path.exists(): - if not self.install_if_missing: - raise exc.AirbyteConnectorNotFoundError( - message="Connector not available and venv does not exist.", - guidance=( - "Please ensure the connector is pre-installed or consider enabling " - "`install_if_missing=True`." - ), + # Store the installed version (or None if not installed) + if not self.reported_version: + self.reported_version = self._get_installed_version() + + original_installed_version = self.reported_version + + reinstalled = False + venv_name = f".venv-{self.name}" + if not self._get_venv_path().exists(): + if not auto_fix: + raise exc.AirbyteConnectorInstallationError( + message="Virtual environment does not exist.", + connector_name=self.name, + context={ + "venv_path": self._get_venv_path(), + }, + ) + + # If the venv path does not exist, install. + self.install() + reinstalled = True + + elif not self._get_connector_path().exists(): + if not auto_fix: + raise exc.AirbyteConnectorInstallationError( + message="Could not locate connector executable within the virtual environment.", + connector_name=self.name, context={ - "connector_name": self.metadata.name, - "venv_name": venv_name, + "connector_path": self._get_connector_path(), }, ) + + # If the connector path does not exist, uninstall and re-install. + # This is sometimes caused by a failed or partial installation. + self.uninstall() self.install() + reinstalled = True + + # By now, everything should be installed. Raise an exception if not. connector_path = self._get_connector_path() if not connector_path.exists(): - raise exc.AirbyteConnectorNotFoundError( - connector_name=self.metadata.name, + raise exc.AirbyteConnectorInstallationError( + message="Connector's executable could not be found within the virtual environment.", + connector_name=self.name, context={ - "venv_name": venv_name, + "connector_path": self._get_connector_path(), }, ) from FileNotFoundError(connector_path) if self.enforce_version: - installed_version = self._get_installed_version() - if installed_version != self.target_version: - # If the version doesn't match, reinstall - self.install() + version_after_reinstall: str | None = None + if self.reported_version != self.target_version: + if auto_fix and not reinstalled: + # If we haven't already reinstalled above, reinstall now. + self.install() + reinstalled = True + + if reinstalled: + version_after_reinstall = self.reported_version # Check the version again - version_after_install = self._get_installed_version() - if version_after_install != self.target_version: + if self.reported_version != self.target_version: raise exc.AirbyteConnectorInstallationError( - connector_name=self.metadata.name, + message="Connector's reported version does not match the target version.", + connector_name=self.name, context={ "venv_name": venv_name, "target_version": self.target_version, - "installed_version": installed_version, - "version_after_install": version_after_install, + "original_installed_version": original_installed_version, + "version_after_reinstall": version_after_reinstall, }, ) @@ -228,33 +352,69 @@ def execute(self, args: list[str]) -> Iterator[str]: yield from stream def get_telemetry_info(self) -> SourceTelemetryInfo: - return SourceTelemetryInfo(self.metadata.name, SourceType.VENV, self.target_version) + return SourceTelemetryInfo( + name=self.name, + type=SourceType.VENV, + version=self.reported_version, + ) class PathExecutor(Executor): - def ensure_installation(self) -> None: + def __init__( + self, + name: str | None = None, + *, + path: Path, + target_version: str | None = None, + ) -> None: + """Initialize a connector executor that runs a connector from a local path. + + If path is simply the name of the connector, it will be expected to exist in the current + PATH or in the current working directory. + """ + self.path: Path = path + name = name or path.name + super().__init__(name=name, target_version=target_version) + + def ensure_installation( + self, + *, + auto_fix: bool = True, + ) -> None: + """Ensure that the connector executable can be found. + + The auto_fix parameter is ignored for this executor type. + """ + _ = auto_fix try: self.execute(["spec"]) except Exception as e: - raise exc.AirbyteConnectorNotFoundError( - connector_name=self.metadata.name, + # TODO: Improve error handling. We should try to distinguish between + # a connector that is not installed and a connector that is not + # working properly. + raise exc.AirbyteConnectorExecutableNotFoundError( + connector_name=self.name, ) from e def install(self) -> NoReturn: raise exc.AirbyteConnectorInstallationError( message="Connector cannot be installed because it is not managed by airbyte-lib.", - connector_name=self.metadata.name, + connector_name=self.name, ) def uninstall(self) -> NoReturn: raise exc.AirbyteConnectorInstallationError( message="Connector cannot be uninstalled because it is not managed by airbyte-lib.", - connector_name=self.metadata.name, + connector_name=self.name, ) def execute(self, args: list[str]) -> Iterator[str]: - with _stream_from_subprocess([self.metadata.name, *args]) as stream: + with _stream_from_subprocess([str(self.path), *args]) as stream: yield from stream def get_telemetry_info(self) -> SourceTelemetryInfo: - return SourceTelemetryInfo(self.metadata.name, SourceType.LOCAL_INSTALL, version=None) + return SourceTelemetryInfo( + str(self.name), + SourceType.LOCAL_INSTALL, + version=self.reported_version, + ) diff --git a/airbyte-lib/airbyte_lib/_factories/connector_factories.py b/airbyte-lib/airbyte_lib/_factories/connector_factories.py index 4dbe8c6f41f0..197ed61142c6 100644 --- a/airbyte-lib/airbyte_lib/_factories/connector_factories.py +++ b/airbyte-lib/airbyte_lib/_factories/connector_factories.py @@ -1,11 +1,13 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. from __future__ import annotations +import shutil +from pathlib import Path from typing import Any -from airbyte_lib._executor import Executor, PathExecutor, VenvExecutor -from airbyte_lib.exceptions import AirbyteLibInputError -from airbyte_lib.registry import get_connector_metadata +from airbyte_lib import exceptions as exc +from airbyte_lib._executor import PathExecutor, VenvExecutor +from airbyte_lib.registry import ConnectorMetadata, get_connector_metadata from airbyte_lib.source import Source @@ -15,7 +17,7 @@ def get_connector( pip_url: str | None = None, config: dict[str, Any] | None = None, *, - use_local_install: bool = False, + local_executable: Path | str | None = None, install_if_missing: bool = True, ) -> Source: """Get a connector by name and version. @@ -29,34 +31,58 @@ def get_connector( connector name. config: connector config - if not provided, you need to set it later via the set_config method. - use_local_install: whether to use a virtual environment to run the connector. If True, the - connector is expected to be available on the path (e.g. installed via pip). If False, - the connector will be installed automatically in a virtual environment. - install_if_missing: whether to install the connector if it is not available locally. This - parameter is ignored if use_local_install is True. + local_executable: If set, the connector will be assumed to already be installed and will be + executed using this path or executable name. Otherwise, the connector will be installed + automatically in a virtual environment. + install_if_missing: Whether to install the connector if it is not available locally. This + parameter is ignored when local_executable is set. """ - metadata = get_connector_metadata(name) - if use_local_install: + if local_executable: if pip_url: - raise AirbyteLibInputError( - message="Param 'pip_url' is not supported when 'use_local_install' is True." + raise exc.AirbyteLibInputError( + message="Param 'pip_url' is not supported when 'local_executable' is set." ) if version: - raise AirbyteLibInputError( - message="Param 'version' is not supported when 'use_local_install' is True." + raise exc.AirbyteLibInputError( + message="Param 'version' is not supported when 'local_executable' is set." ) - executor: Executor = PathExecutor( - metadata=metadata, - target_version=version, - ) - else: - executor = VenvExecutor( - metadata=metadata, - target_version=version, - install_if_missing=install_if_missing, - pip_url=pip_url, + if isinstance(local_executable, str): + if "/" in local_executable or "\\" in local_executable: + # Assume this is a path + local_executable = Path(local_executable).absolute() + else: + which_executable = shutil.which(local_executable) + if which_executable is None: + raise FileNotFoundError(local_executable) + local_executable = Path(which_executable).absolute() + + return Source( + name=name, + config=config, + executor=PathExecutor( + name=name, + path=local_executable, + ), ) + + metadata: ConnectorMetadata | None = None + try: + metadata = get_connector_metadata(name) + except exc.AirbyteConnectorNotRegisteredError: + if not pip_url: + # We don't have a pip url or registry entry, so we can't install the connector + raise + + executor = VenvExecutor( + name=name, + metadata=metadata, + target_version=version, + pip_url=pip_url, + ) + if install_if_missing: + executor.ensure_installation() + return Source( executor=executor, name=name, diff --git a/airbyte-lib/airbyte_lib/exceptions.py b/airbyte-lib/airbyte_lib/exceptions.py index 3c6336d03103..934e936d4d48 100644 --- a/airbyte-lib/airbyte_lib/exceptions.py +++ b/airbyte-lib/airbyte_lib/exceptions.py @@ -174,6 +174,14 @@ class AirbyteConnectorRegistryError(AirbyteError): """Error when accessing the connector registry.""" +@dataclass +class AirbyteConnectorNotRegisteredError(AirbyteConnectorRegistryError): + """Connector not found in registry.""" + + connector_name: str | None = None + guidance = "Please double check the connector name." + + # Connector Errors @@ -184,8 +192,8 @@ class AirbyteConnectorError(AirbyteError): connector_name: str | None = None -class AirbyteConnectorNotFoundError(AirbyteConnectorError): - """Connector not found.""" +class AirbyteConnectorExecutableNotFoundError(AirbyteConnectorError): + """Connector executable not found.""" class AirbyteConnectorInstallationError(AirbyteConnectorError): diff --git a/airbyte-lib/airbyte_lib/registry.py b/airbyte-lib/airbyte_lib/registry.py index bd030a867ff0..a7faf64eb919 100644 --- a/airbyte-lib/airbyte_lib/registry.py +++ b/airbyte-lib/airbyte_lib/registry.py @@ -3,6 +3,7 @@ import json import os +from copy import copy from dataclasses import dataclass from pathlib import Path @@ -12,32 +13,60 @@ from airbyte_lib.version import get_version +__cache: dict[str, ConnectorMetadata] | None = None + + +REGISTRY_ENV_VAR = "AIRBYTE_LOCAL_REGISTRY" +REGISTRY_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json" + + @dataclass class ConnectorMetadata: name: str latest_available_version: str -_cache: dict[str, ConnectorMetadata] | None = None +def _get_registry_url() -> str: + if REGISTRY_ENV_VAR in os.environ: + return str(os.environ.get(REGISTRY_ENV_VAR)) -REGISTRY_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json" + return REGISTRY_URL -def _update_cache() -> None: - global _cache - if os.environ.get("AIRBYTE_LOCAL_REGISTRY"): - with Path(str(os.environ.get("AIRBYTE_LOCAL_REGISTRY"))).open() as f: - data = json.load(f) - else: +def _get_registry_cache(*, force_refresh: bool = False) -> dict[str, ConnectorMetadata]: + """Return the registry cache.""" + global __cache + if __cache and not force_refresh: + return __cache + + registry_url = _get_registry_url() + if registry_url.startswith("http"): response = requests.get( - REGISTRY_URL, headers={"User-Agent": f"airbyte-lib-{get_version()}"} + registry_url, headers={"User-Agent": f"airbyte-lib-{get_version()}"} ) response.raise_for_status() data = response.json() - _cache = {} + else: + # Assume local file + with Path(registry_url).open() as f: + data = json.load(f) + + new_cache: dict[str, ConnectorMetadata] = {} + for connector in data["sources"]: name = connector["dockerRepository"].replace("airbyte/", "") - _cache[name] = ConnectorMetadata(name, connector["dockerImageTag"]) + new_cache[name] = ConnectorMetadata(name, connector["dockerImageTag"]) + + if len(new_cache) == 0: + raise exc.AirbyteLibInternalError( + message="Connector registry is empty.", + context={ + "registry_url": _get_registry_url(), + }, + ) + + __cache = new_cache + return __cache def get_connector_metadata(name: str) -> ConnectorMetadata: @@ -45,14 +74,20 @@ def get_connector_metadata(name: str) -> ConnectorMetadata: If the cache is empty, populate by calling update_cache. """ - if not _cache: - _update_cache() - if not _cache or name not in _cache: - raise exc.AirbyteLibInputError( - message="Connector name not found in registry.", - guidance="Please double check the connector name.", + cache = copy(_get_registry_cache()) + if not cache: + raise exc.AirbyteLibInternalError( + message="Connector registry could not be loaded.", + context={ + "registry_url": _get_registry_url(), + }, + ) + if name not in cache: + raise exc.AirbyteConnectorNotRegisteredError( + connector_name=name, context={ - "connector_name": name, + "registry_url": _get_registry_url(), + "available_connectors": sorted(cache.keys()), }, ) - return _cache[name] + return cache[name] diff --git a/airbyte-lib/airbyte_lib/source.py b/airbyte-lib/airbyte_lib/source.py index ec37e11791dd..4db25b3afae0 100644 --- a/airbyte-lib/airbyte_lib/source.py +++ b/airbyte-lib/airbyte_lib/source.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any import jsonschema +import yaml from airbyte_protocol.models import ( AirbyteCatalog, @@ -68,7 +69,13 @@ def __init__( name: str, config: dict[str, Any] | None = None, streams: list[str] | None = None, + *, + validate: bool = False, ) -> None: + """Initialize the source. + + If config is provided, it will be validated against the spec if validate is True. + """ self._processed_records = 0 self.executor = executor self.name = name @@ -79,7 +86,7 @@ def __init__( self._spec: ConnectorSpecification | None = None self._selected_stream_names: list[str] | None = None if config is not None: - self.set_config(config) + self.set_config(config, validate=validate) if streams is not None: self.set_streams(streams) @@ -102,8 +109,22 @@ def set_streams(self, streams: list[str]) -> None: ) self._selected_stream_names = streams - def set_config(self, config: dict[str, Any]) -> None: - self._validate_config(config) + def set_config( + self, + config: dict[str, Any], + *, + validate: bool = False, + ) -> None: + """Set the config for the connector. + + If validate is True, raise an exception if the config fails validation. + + If validate is False, validation will be deferred until check() or validate_config() + is called. + """ + if validate: + self.validate_config(config) + self._config_dict = config @property @@ -131,9 +152,13 @@ def _discover(self) -> AirbyteCatalog: log_text=self._last_log_messages, ) - def _validate_config(self, config: dict[str, Any]) -> None: - """Validate the config against the spec.""" + def validate_config(self, config: dict[str, Any] | None = None) -> None: + """Validate the config against the spec. + + If config is not provided, the already-set config will be validated. + """ spec = self._get_spec(force_refresh=False) + config = self._config if config is None else config jsonschema.validate(config, spec.connectionSpecification) def get_available_streams(self) -> list[str]: @@ -161,6 +186,21 @@ def _get_spec(self, *, force_refresh: bool = False) -> ConnectorSpecification: log_text=self._last_log_messages, ) + @property + def _yaml_spec(self) -> str: + """Get the spec as a yaml string. + + For now, the primary use case is for writing and debugging a valid config for a source. + + This is private for now because we probably want better polish before exposing this + as a stable interface. This will also get easier when we have docs links with this info + for each connector. + """ + spec_obj: ConnectorSpecification = self._get_spec() + spec_dict = spec_obj.dict(exclude_unset=True) + # convert to a yaml string + return yaml.dump(spec_dict) + @property def discovered_catalog(self) -> AirbyteCatalog: """Get the raw catalog for the given streams. @@ -248,17 +288,23 @@ def check(self) -> None: * Make sure the subprocess is killed when the function returns. """ with as_temp_files([self._config]) as [config_file]: - for msg in self._execute(["check", "--config", config_file]): - if msg.type == Type.CONNECTION_STATUS and msg.connectionStatus: - if msg.connectionStatus.status != Status.FAILED: - return # Success! - - raise exc.AirbyteConnectorCheckFailedError( - context={ - "message": msg.connectionStatus.message, - } - ) - raise exc.AirbyteConnectorCheckFailedError(log_text=self._last_log_messages) + try: + for msg in self._execute(["check", "--config", config_file]): + if msg.type == Type.CONNECTION_STATUS and msg.connectionStatus: + if msg.connectionStatus.status != Status.FAILED: + return # Success! + + raise exc.AirbyteConnectorCheckFailedError( + context={ + "message": msg.connectionStatus.message, + } + ) + raise exc.AirbyteConnectorCheckFailedError(log_text=self._last_log_messages) + except exc.AirbyteConnectorReadError as ex: + raise exc.AirbyteConnectorCheckFailedError( + message="The connector failed to check the connection.", + log_text=ex.log_text, + ) from ex def install(self) -> None: """Install the connector if it is not yet installed.""" @@ -338,7 +384,8 @@ def _execute(self, args: list[str]) -> Iterator[AirbyteMessage]: * Read the output line by line of the subprocess and serialize them AirbyteMessage objects. Drop if not valid. """ - self.executor.ensure_installation() + # Fail early if the connector is not installed. + self.executor.ensure_installation(auto_fix=False) try: self._last_log_messages = [] diff --git a/airbyte-lib/airbyte_lib/validate.py b/airbyte-lib/airbyte_lib/validate.py index 75eab7e3fd39..551960ed5250 100644 --- a/airbyte-lib/airbyte_lib/validate.py +++ b/airbyte-lib/airbyte_lib/validate.py @@ -42,11 +42,16 @@ def _parse_args() -> argparse.Namespace: def _run_subprocess_and_raise_on_failure(args: list[str]) -> None: - result = subprocess.run(args, check=False) + result = subprocess.run( + args, + check=False, + stderr=subprocess.PIPE, + ) if result.returncode != 0: raise exc.AirbyteSubprocessFailedError( run_args=args, exit_code=result.returncode, + log_text=result.stderr.decode("utf-8"), ) @@ -55,7 +60,8 @@ def full_tests(connector_name: str, sample_config: str) -> None: source = ab.get_connector( # TODO: FIXME: noqa: SIM115, PTH123 connector_name, - config=json.load(open(sample_config)), # noqa: SIM115, PTH123 + config=json.load(open(sample_config)), # noqa: SIM115, PTH123, + install_if_missing=False, ) print("Running check...") diff --git a/airbyte-lib/docs/generated/airbyte_lib.html b/airbyte-lib/docs/generated/airbyte_lib.html index c8d9f47128ea..ba7c11e54e3d 100644 --- a/airbyte-lib/docs/generated/airbyte_lib.html +++ b/airbyte-lib/docs/generated/airbyte_lib.html @@ -254,7 +254,7 @@
Inherited Members
def - get_connector( name: str, version: str | None = None, pip_url: str | None = None, config: dict[str, typing.Any] | None = None, *, use_local_install: bool = False, install_if_missing: bool = True) -> Source: + get_connector( name: str, version: str | None = None, pip_url: str | None = None, config: dict[str, typing.Any] | None = None, *, local_executable: pathlib.Path | str | None = None, install_if_missing: bool = True) -> Source:
@@ -271,11 +271,11 @@
Inherited Members
connector name. config: connector config - if not provided, you need to set it later via the set_config method. - use_local_install: whether to use a virtual environment to run the connector. If True, the - connector is expected to be available on the path (e.g. installed via pip). If False, - the connector will be installed automatically in a virtual environment. - install_if_missing: whether to install the connector if it is not available locally. This - parameter is ignored if use_local_install is True.

+ local_executable: If set, the connector will be assumed to already be installed and will be + executed using this path or executable name. Otherwise, the connector will be installed + automatically in a virtual environment. + install_if_missing: Whether to install the connector if it is not available locally. This + parameter is ignored when local_executable is set.

@@ -409,13 +409,17 @@
Inherited Members
- Source( executor: airbyte_lib._executor.Executor, name: str, config: dict[str, typing.Any] | None = None, streams: list[str] | None = None) + Source( executor: airbyte_lib._executor.Executor, name: str, config: dict[str, typing.Any] | None = None, streams: list[str] | None = None, *, validate: bool = False)
- +

Initialize the source.

+ +

If config is provided, it will be validated against the spec if validate is True.

+
+
@@ -465,13 +469,37 @@
Inherited Members
def - set_config(self, config: dict[str, typing.Any]) -> None: + set_config(self, config: dict[str, typing.Any], *, validate: bool = False) -> None:
+

Set the config for the connector.

+ +

If validate is True, raise an exception if the config fails validation.

+ +

If validate is False, validation will be deferred until check() or validate_config() +is called.

+
+ + +
+
+
+ + def + validate_config(self, config: dict[str, typing.Any] | None = None) -> None: + + +
+ +

Validate the config against the spec.

+ +

If config is not provided, the already-set config will be validated.

+
+
diff --git a/airbyte-lib/tests/integration_tests/fixtures/source-test/source_test/__init__.py b/airbyte-lib/tests/integration_tests/fixtures/source-test/source_test/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-lib/tests/integration_tests/test_install.py b/airbyte-lib/tests/integration_tests/test_install.py new file mode 100644 index 000000000000..c93801489f37 --- /dev/null +++ b/airbyte-lib/tests/integration_tests/test_install.py @@ -0,0 +1,23 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from gettext import install +import pytest + +from airbyte_lib._factories.connector_factories import get_connector +from airbyte_lib import exceptions as exc + + +def test_install_failure_log_pypi(): + """Test that the install log is created and contains the expected content.""" + with pytest.raises(exc.AirbyteConnectorNotRegisteredError): + source = get_connector("source-not-found") + + with pytest.raises(exc.AirbyteConnectorInstallationError) as exc_info: + source = get_connector( + "source-not-found", + pip_url="https://pypi.org/project/airbyte-not-found", + install_if_missing=True, + ) + + # Check that the stderr log contains the expected content from a failed pip install + assert 'Could not install requirement' in str(exc_info.value.__cause__.log_text) diff --git a/airbyte-lib/tests/integration_tests/test_integration.py b/airbyte-lib/tests/integration_tests/test_integration.py index 5dcd1f54e649..5d5586197577 100644 --- a/airbyte-lib/tests/integration_tests/test_integration.py +++ b/airbyte-lib/tests/integration_tests/test_integration.py @@ -3,6 +3,7 @@ from collections.abc import Mapping import os import shutil +import subprocess from typing import Any from unittest.mock import Mock, call, patch import tempfile @@ -17,7 +18,7 @@ import pytest from airbyte_lib.caches import PostgresCache, PostgresCacheConfig -from airbyte_lib.registry import _update_cache +from airbyte_lib import registry from airbyte_lib.version import get_version from airbyte_lib.results import ReadResult from airbyte_lib.datasets import CachedDataset, LazyDataset, SQLDataset @@ -28,24 +29,32 @@ import ulid -@pytest.fixture(scope="module", autouse=True) +LOCAL_TEST_REGISTRY_URL = "./tests/integration_tests/fixtures/registry.json" + + +@pytest.fixture(scope="package", autouse=True) def prepare_test_env(): """ Prepare test environment. This will pre-install the test source from the fixtures array and set the environment variable to use the local json file as registry. """ - if os.path.exists(".venv-source-test"): - shutil.rmtree(".venv-source-test") + venv_dir = ".venv-source-test" + if os.path.exists(venv_dir): + shutil.rmtree(venv_dir) - os.system("python -m venv .venv-source-test") - os.system(".venv-source-test/bin/pip install -e ./tests/integration_tests/fixtures/source-test") + subprocess.run(["python", "-m", "venv", venv_dir], check=True) + subprocess.run([f"{venv_dir}/bin/pip", "install", "-e", "./tests/integration_tests/fixtures/source-test"], check=True) - os.environ["AIRBYTE_LOCAL_REGISTRY"] = "./tests/integration_tests/fixtures/registry.json" + os.environ["AIRBYTE_LOCAL_REGISTRY"] = LOCAL_TEST_REGISTRY_URL os.environ["DO_NOT_TRACK"] = "true" + # Force-refresh the registry cache + _ = registry._get_registry_cache(force_refresh=True) + yield shutil.rmtree(".venv-source-test") + @pytest.fixture def expected_test_stream_data() -> dict[str, list[dict[str, str | int]]]: return { @@ -58,15 +67,50 @@ def expected_test_stream_data() -> dict[str, list[dict[str, str | int]]]: ], } -def test_list_streams(expected_test_stream_data: dict[str, list[dict[str, str | int]]]): - source = ab.get_connector("source-test", config={"apiKey": "test"}) +def test_registry_get(): + assert registry._get_registry_url() == LOCAL_TEST_REGISTRY_URL + metadata = registry.get_connector_metadata("source-test") + assert metadata.name == "source-test" + assert metadata.latest_available_version == "0.0.1" + + +def test_list_streams(expected_test_stream_data: dict[str, list[dict[str, str | int]]]): + source = ab.get_connector( + "source-test", config={"apiKey": "test"}, install_if_missing=False + ) assert source.get_available_streams() == list(expected_test_stream_data.keys()) def test_invalid_config(): - with pytest.raises(Exception): - ab.get_connector("source-test", config={"apiKey": 1234}) + source = ab.get_connector( + "source-test", config={"apiKey": 1234}, install_if_missing=False + ) + with pytest.raises(exc.AirbyteConnectorCheckFailedError): + source.check() + + +def test_ensure_installation_detection(): + """Assert that install isn't called, since the connector is already installed by the fixture.""" + with patch("airbyte_lib._executor.VenvExecutor.install") as mock_venv_install, \ + patch("airbyte_lib.source.Source.install") as mock_source_install, \ + patch("airbyte_lib._executor.VenvExecutor.ensure_installation") as mock_ensure_installed: + source = ab.get_connector( + "source-test", + config={"apiKey": 1234}, + pip_url="https://pypi.org/project/airbyte-not-found", + install_if_missing=True, + ) + assert mock_ensure_installed.call_count == 1 + assert not mock_venv_install.called + assert not mock_source_install.called + + +def test_source_yaml_spec(): + source = ab.get_connector( + "source-test", config={"apiKey": 1234}, install_if_missing=False + ) + assert source._yaml_spec.startswith("connectionSpecification:\n $schema:") def test_non_existing_connector(): @@ -93,22 +137,35 @@ def test_version_enforcement(raises, latest_available_version, requested_version In this test, the actually installed version is 0.0.1 """ - _update_cache() - from airbyte_lib.registry import _cache - _cache["source-test"].latest_available_version = latest_available_version - if raises: - with pytest.raises(Exception): - ab.get_connector("source-test", version=requested_version, config={"apiKey": "abc"}) - else: - ab.get_connector("source-test", version=requested_version, config={"apiKey": "abc"}) - - # reset - _cache["source-test"].latest_available_version = "0.0.1" + patched_entry = registry.ConnectorMetadata( + name="source-test", latest_available_version=latest_available_version + ) + with patch.dict("airbyte_lib.registry.__cache", {"source-test": patched_entry}, clear=False): + if raises: + with pytest.raises(Exception): + source = ab.get_connector( + "source-test", + version=requested_version, + config={"apiKey": "abc"}, + install_if_missing=False, + ) + source.executor.ensure_installation(auto_fix=False) + else: + source = ab.get_connector( + "source-test", + version=requested_version, + config={"apiKey": "abc"}, + install_if_missing=False, + ) + source.executor.ensure_installation(auto_fix=False) def test_check(): - source = ab.get_connector("source-test", config={"apiKey": "test"}) - + source = ab.get_connector( + "source-test", + config={"apiKey": "test"}, + install_if_missing=False, + ) source.check() @@ -141,7 +198,7 @@ def assert_cache_data(expected_test_stream_data: dict[str, list[dict[str, str | pd.DataFrame(expected_test_stream_data[stream_name]), check_dtype=False, ) - + # validate that the cache doesn't contain any other streams if streams: assert len(list(cache.__iter__())) == len(streams) @@ -459,6 +516,15 @@ def test_sync_with_merge_to_postgres(new_pg_cache_config: PostgresCacheConfig, e check_dtype=False, ) + +def test_airbyte_lib_version() -> None: + assert get_version() + assert isinstance(get_version(), str) + + # Ensure the version is a valid semantic version (x.y.z or x.y.z.alpha0) + assert 3 <= len(get_version().split(".")) <= 4 + + @patch.dict('os.environ', {'DO_NOT_TRACK': ''}) @patch('airbyte_lib.telemetry.requests') @patch('airbyte_lib.telemetry.datetime') @@ -601,27 +667,48 @@ def test_failing_path_connector(): ab.get_connector("source-test", config={"apiKey": "test"}, use_local_install=True) def test_succeeding_path_connector(): - old_path = os.environ["PATH"] + new_path = f"{os.path.abspath('.venv-source-test/bin')}:{os.environ['PATH']}" + + # Patch the PATH env var to include the test venv bin folder + with patch.dict(os.environ, {"PATH": new_path}): + source = ab.get_connector( + "source-test", + config={"apiKey": "test"}, + local_executable="source-test", + ) + source.check() - # set path to include the test venv bin folder - os.environ["PATH"] = f"{os.path.abspath('.venv-source-test/bin')}:{os.environ['PATH']}" - source = ab.get_connector("source-test", config={"apiKey": "test"}, use_local_install=True) - source.check() +def test_install_uninstall(): + with tempfile.TemporaryDirectory() as temp_dir: + source = ab.get_connector( + "source-test", + pip_url="./tests/integration_tests/fixtures/source-test", + config={"apiKey": "test"}, + install_if_missing=False, + ) - os.environ["PATH"] = old_path + # Override the install root to avoid conflicts with the test fixture + install_root = Path(temp_dir) + source.executor.install_root = install_root -def test_install_uninstall(): - source = ab.get_connector("source-test", pip_url="./tests/integration_tests/fixtures/source-test", config={"apiKey": "test"}, install_if_missing=False) + # assert that the venv is gone + assert not os.path.exists(install_root / ".venv-source-test") - source.uninstall() + # use which to check if the executable is available + assert shutil.which("source-test") is None - # assert that the venv is gone - assert not os.path.exists(".venv-source-test") + # assert that the connector is not available + with pytest.raises(Exception): + source.check() + + source.install() + + assert os.path.exists(install_root / ".venv-source-test") + assert os.path.exists(install_root / ".venv-source-test/bin/source-test") - # assert that the connector is not available - with pytest.raises(Exception): source.check() - source.install() + source.uninstall() - source.check() + assert not os.path.exists(install_root / ".venv-source-test") + assert not os.path.exists(install_root / ".venv-source-test/bin/source-test") From 98d760f6f133e0abd82a2edce95f80db339fd237 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Mon, 29 Jan 2024 22:18:29 -0800 Subject: [PATCH 09/96] Revert Default Cloud Version (#34646) --- .../connectors/destination-postgres-strict-encrypt/metadata.yaml | 1 + .../connectors/destination-postgres/metadata.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml index c9136954c393..6403177cee3a 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml @@ -16,6 +16,7 @@ data: registries: cloud: enabled: false + dockerImageTag: 0.6.0 oss: enabled: false releaseStage: alpha diff --git a/airbyte-integrations/connectors/destination-postgres/metadata.yaml b/airbyte-integrations/connectors/destination-postgres/metadata.yaml index 26b08f6fb1b3..730c511df3b8 100644 --- a/airbyte-integrations/connectors/destination-postgres/metadata.yaml +++ b/airbyte-integrations/connectors/destination-postgres/metadata.yaml @@ -19,6 +19,7 @@ data: registries: cloud: dockerRepository: airbyte/destination-postgres-strict-encrypt + dockerImageTag: 0.6.0 enabled: true oss: enabled: true From 6408023b6f3c56ce6ec3d984117901b02dc9c7df Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Mon, 29 Jan 2024 22:39:26 -0800 Subject: [PATCH 10/96] AirbyteLib: Progress Printer (#34588) --- airbyte-lib/airbyte_lib/_processors.py | 17 +- airbyte-lib/airbyte_lib/progress.py | 320 ++++++++++++++++++ airbyte-lib/airbyte_lib/source.py | 5 +- .../docs/generated/airbyte_lib/caches.html | 2 +- airbyte-lib/examples/run_faker.py | 31 ++ airbyte-lib/poetry.lock | 69 +++- airbyte-lib/pyproject.toml | 2 + airbyte-lib/tests/conftest.py | 28 ++ airbyte-lib/tests/unit_tests/test_progress.py | 174 ++++++++++ 9 files changed, 640 insertions(+), 8 deletions(-) create mode 100644 airbyte-lib/airbyte_lib/progress.py create mode 100644 airbyte-lib/examples/run_faker.py create mode 100644 airbyte-lib/tests/unit_tests/test_progress.py diff --git a/airbyte-lib/airbyte_lib/_processors.py b/airbyte-lib/airbyte_lib/_processors.py index d5eba9f2c00f..4418463a8a1d 100644 --- a/airbyte-lib/airbyte_lib/_processors.py +++ b/airbyte-lib/airbyte_lib/_processors.py @@ -32,6 +32,7 @@ from airbyte_lib import exceptions as exc from airbyte_lib._util import protocol_util # Internal utility functions +from airbyte_lib.progress import progress if TYPE_CHECKING: @@ -40,7 +41,7 @@ from airbyte_lib.config import CacheConfigBase -DEFAULT_BATCH_SIZE = 10000 +DEFAULT_BATCH_SIZE = 10_000 class BatchHandle: @@ -95,7 +96,7 @@ def register_source( For now, only one source at a time is supported. If this method is called multiple times, the last call will overwrite the previous one. - TODO: Expand this to handle mutliple sources. + TODO: Expand this to handle multiple sources. """ _ = source_name self.source_catalog = incoming_source_catalog @@ -157,6 +158,7 @@ def process_airbyte_messages( if len(stream_batch) >= max_batch_size: record_batch = pa.Table.from_pylist(stream_batch) self._process_batch(stream_name, record_batch) + progress.log_batch_written(stream_name, len(stream_batch)) stream_batch.clear() elif message.type is Type.STATE: @@ -180,14 +182,16 @@ def process_airbyte_messages( ) # We are at the end of the stream. Process whatever else is queued. - for stream_name, batch in stream_batches.items(): - if batch: - record_batch = pa.Table.from_pylist(batch) + for stream_name, stream_batch in stream_batches.items(): + if stream_batch: + record_batch = pa.Table.from_pylist(stream_batch) self._process_batch(stream_name, record_batch) + progress.log_batch_written(stream_name, len(stream_batch)) # Finalize any pending batches for stream_name in list(self._pending_batches.keys()): self._finalize_batches(stream_name) + progress.log_stream_finalized(stream_name) @final def _process_batch( @@ -287,7 +291,10 @@ def _finalizing_batches( state_messages_to_finalize = self._pending_state_messages[stream_name].copy() self._pending_batches[stream_name].clear() self._pending_state_messages[stream_name].clear() + + progress.log_batches_finalizing(stream_name, len(batches_to_finalize)) yield batches_to_finalize + progress.log_batches_finalized(stream_name, len(batches_to_finalize)) self._finalized_batches[stream_name].update(batches_to_finalize) self._finalized_state_messages[stream_name] += state_messages_to_finalize diff --git a/airbyte-lib/airbyte_lib/progress.py b/airbyte-lib/airbyte_lib/progress.py new file mode 100644 index 000000000000..d1b7c5355fe1 --- /dev/null +++ b/airbyte-lib/airbyte_lib/progress.py @@ -0,0 +1,320 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""A simple progress bar for the command line and IPython notebooks.""" +from __future__ import annotations + +import datetime +import math +import time +from contextlib import suppress +from typing import cast + +from rich.errors import LiveError +from rich.live import Live as RichLive +from rich.markdown import Markdown as RichMarkdown + + +try: + IS_NOTEBOOK = True + from IPython import display as ipy_display + +except ImportError: + ipy_display = None + IS_NOTEBOOK = False + + +MAX_UPDATE_FREQUENCY = 1000 +"""The max number of records to read before updating the progress bar.""" + + +def _to_time_str(timestamp: float) -> str: + """Convert a timestamp float to a local time string. + + For now, we'll just use UTC to avoid breaking tests. In the future, we should + return a local time string. + """ + datetime_obj = datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc) + # TODO: Uncomment this line when we can get tests to properly account for local timezones. + # For now, we'll just use UTC to avoid breaking tests. + # datetime_obj = datetime_obj.astimezone() + return datetime_obj.strftime("%H:%M:%S") + + +def _get_elapsed_time_str(seconds: int) -> str: + """Return duration as a string. + + Seconds are included until 10 minutes is exceeded. + Minutes are always included after 1 minute elapsed. + Hours are always included after 1 hour elapsed. + """ + if seconds <= 60: # noqa: PLR2004 # Magic numbers OK here. + return f"{seconds} seconds" + + if seconds < 60 * 10: + minutes = seconds // 60 + seconds = seconds % 60 + return f"{minutes}min {seconds}s" + + if seconds < 60 * 60: + minutes = seconds // 60 + seconds = seconds % 60 + return f"{minutes}min" + + hours = seconds // (60 * 60) + minutes = (seconds % (60 * 60)) // 60 + return f"{hours}hr {minutes}min" + + +class ReadProgress: + """A simple progress bar for the command line and IPython notebooks.""" + + def __init__(self) -> None: + """Initialize the progress tracker.""" + # Streams expected (for progress bar) + self.num_streams_expected = 0 + + # Reads + self.read_start_time = time.time() + self.read_end_time: float | None = None + self.total_records_read = 0 + + # Writes + self.total_records_written = 0 + self.total_batches_written = 0 + self.written_stream_names: set[str] = set() + + # Finalization + self.finalize_start_time: float | None = None + self.finalize_end_time: float | None = None + self.total_records_finalized = 0 + self.total_batches_finalized = 0 + self.finalized_stream_names: set[str] = set() + + self.last_update_time: float | None = None + + self.rich_view: RichLive | None = None + if not IS_NOTEBOOK: + # If we're in a terminal, use a Rich view to display the progress updates. + self.rich_view = RichLive() + try: + self.rich_view.start() + except LiveError: + self.rich_view = None + + def __del__(self) -> None: + """Close the Rich view.""" + if self.rich_view: + with suppress(Exception): + self.rich_view.stop() + + def log_success(self) -> None: + """Log success and stop tracking progress.""" + if self.finalize_end_time is None: + # If we haven't already finalized, do so now. + + self.finalize_end_time = time.time() + + self.update_display(force_refresh=True) + if self.rich_view: + with suppress(Exception): + self.rich_view.stop() + + def reset(self, num_streams_expected: int) -> None: + """Reset the progress tracker.""" + # Streams expected (for progress bar) + self.num_streams_expected = num_streams_expected + + # Reads + self.read_start_time = time.time() + self.read_end_time = None + self.total_records_read = 0 + + # Writes + self.total_records_written = 0 + self.total_batches_written = 0 + self.written_stream_names = set() + + # Finalization + self.finalize_start_time = None + self.finalize_end_time = None + self.total_records_finalized = 0 + self.total_batches_finalized = 0 + self.finalized_stream_names = set() + + @property + def elapsed_seconds(self) -> int: + """Return the number of seconds elapsed since the read operation started.""" + if self.finalize_end_time: + return int(self.finalize_end_time - self.read_start_time) + + return int(time.time() - self.read_start_time) + + @property + def elapsed_time_string(self) -> str: + """Return duration as a string.""" + return _get_elapsed_time_str(self.elapsed_seconds) + + @property + def elapsed_seconds_since_last_update(self) -> float | None: + """Return the number of seconds elapsed since the last update.""" + if self.last_update_time is None: + return None + + return time.time() - self.last_update_time + + @property + def elapsed_read_seconds(self) -> int: + """Return the number of seconds elapsed since the read operation started.""" + if self.read_end_time is None: + return int(time.time() - self.read_start_time) + + return int(self.read_end_time - self.read_start_time) + + @property + def elapsed_read_time_string(self) -> str: + """Return duration as a string.""" + return _get_elapsed_time_str(self.elapsed_read_seconds) + + @property + def elapsed_finalization_seconds(self) -> int: + """Return the number of seconds elapsed since the read operation started.""" + if self.finalize_start_time is None: + return 0 + if self.finalize_end_time is None: + return int(time.time() - self.finalize_start_time) + return int(self.finalize_end_time - self.finalize_start_time) + + @property + def elapsed_finalization_time_str(self) -> str: + """Return duration as a string.""" + return _get_elapsed_time_str(self.elapsed_finalization_seconds) + + def log_records_read(self, new_total_count: int) -> None: + """Load a number of records read.""" + self.total_records_read = new_total_count + + # This is some math to make updates adaptive to the scale of records read. + # We want to update the display more often when the count is low, and less + # often when the count is high. + updated_period = min( + MAX_UPDATE_FREQUENCY, 10 ** math.floor(math.log10(self.total_records_read) / 4) + ) + if self.total_records_read % updated_period != 0: + return + + self.update_display() + + def log_batch_written(self, stream_name: str, batch_size: int) -> None: + """Log that a batch has been written. + + Args: + stream_name: The name of the stream. + batch_size: The number of records in the batch. + """ + self.total_records_written += batch_size + self.total_batches_written += 1 + self.written_stream_names.add(stream_name) + self.update_display() + + def log_batches_finalizing(self, stream_name: str, num_batches: int) -> None: + """Log that batch are ready to be finalized. + + In our current implementation, we ignore the stream name and number of batches. + We just use this as a signal that we're finished reading and have begun to + finalize any accumulated batches. + """ + _ = stream_name, num_batches # unused for now + if self.finalize_start_time is None: + self.read_end_time = time.time() + self.finalize_start_time = self.read_end_time + + self.update_display(force_refresh=True) + + def log_batches_finalized(self, stream_name: str, num_batches: int) -> None: + """Log that a batch has been finalized.""" + _ = stream_name # unused for now + self.total_batches_finalized += num_batches + self.update_display(force_refresh=True) + + def log_stream_finalized(self, stream_name: str) -> None: + """Log that a stream has been finalized.""" + self.finalized_stream_names.add(stream_name) + if len(self.finalized_stream_names) == self.num_streams_expected: + self.log_success() + + self.update_display(force_refresh=True) + + def update_display(self, *, force_refresh: bool = False) -> None: + """Update the display.""" + # Don't update more than twice per second unless force_refresh is True. + if ( + not force_refresh + and self.last_update_time # if not set, then we definitely need to update + and cast(float, self.elapsed_seconds_since_last_update) < 0.5 # noqa: PLR2004 + ): + return + + status_message = self._get_status_message() + + if IS_NOTEBOOK: + # We're in a notebook so use the IPython display. + ipy_display.clear_output(wait=True) + ipy_display.display(ipy_display.Markdown(status_message)) + + elif self.rich_view is not None: + self.rich_view.update(RichMarkdown(status_message)) + + self.last_update_time = time.time() + + def _get_status_message(self) -> str: + """Compile and return a status message.""" + # Format start time as a friendly string in local timezone: + start_time_str = _to_time_str(self.read_start_time) + records_per_second: float = 0.0 + if self.elapsed_read_seconds > 0: + records_per_second = round(self.total_records_read / self.elapsed_read_seconds, 1) + status_message = ( + f"## Read Progress\n\n" + f"Started reading at {start_time_str}.\n\n" + f"Read **{self.total_records_read:,}** records " + f"over **{self.elapsed_read_time_string}** " + f"({records_per_second:,} records / second).\n\n" + ) + if self.total_records_written > 0: + status_message += ( + f"Wrote **{self.total_records_written:,}** records " + f"over {self.total_batches_written:,} batches.\n\n" + ) + if self.read_end_time is not None: + read_end_time_str = _to_time_str(self.read_end_time) + status_message += f"Finished reading at {read_end_time_str}.\n\n" + if self.finalize_start_time is not None: + finalize_start_time_str = _to_time_str(self.finalize_start_time) + status_message += f"Started finalizing streams at {finalize_start_time_str}.\n\n" + status_message += ( + f"Finalized **{self.total_batches_finalized}** batches " + f"over {self.elapsed_finalization_time_str}.\n\n" + ) + if self.finalized_stream_names: + status_message += ( + f"Completed {len(self.finalized_stream_names)} " + + (f"out of {self.num_streams_expected} " if self.num_streams_expected else "") + + "streams:\n\n" + ) + for stream_name in self.finalized_stream_names: + status_message += f" - {stream_name}\n" + + status_message += "\n\n" + + if self.finalize_end_time is not None: + completion_time_str = _to_time_str(self.finalize_end_time) + status_message += ( + f"Completed writing at {completion_time_str}. " + f"Total time elapsed: {self.elapsed_time_string}\n\n" + ) + status_message += "\n------------------------------------------------\n" + + return status_message + + +progress = ReadProgress() diff --git a/airbyte-lib/airbyte_lib/source.py b/airbyte-lib/airbyte_lib/source.py index 4db25b3afae0..120d25adba05 100644 --- a/airbyte-lib/airbyte_lib/source.py +++ b/airbyte-lib/airbyte_lib/source.py @@ -26,6 +26,7 @@ from airbyte_lib._factories.cache_factories import get_default_cache from airbyte_lib._util import protocol_util # Internal utility functions from airbyte_lib.datasets._lazy import LazyDataset +from airbyte_lib.progress import progress from airbyte_lib.results import ReadResult from airbyte_lib.telemetry import ( CacheTelemetryInfo, @@ -76,7 +77,6 @@ def __init__( If config is provided, it will be validated against the spec if validate is True. """ - self._processed_records = 0 self.executor = executor self.name = name self._processed_records = 0 @@ -408,9 +408,12 @@ def _tally_records( ) -> Generator[AirbyteRecordMessage, Any, None]: """This method simply tallies the number of records processed and yields the messages.""" self._processed_records = 0 # Reset the counter before we start + progress.reset(len(self._selected_stream_names or [])) + for message in messages: self._processed_records += 1 yield message + progress.log_records_read(self._processed_records) def read(self, cache: SQLCacheBase | None = None) -> ReadResult: if cache is None: diff --git a/airbyte-lib/docs/generated/airbyte_lib/caches.html b/airbyte-lib/docs/generated/airbyte_lib/caches.html index bbebc9ac9f46..b39a5230e658 100644 --- a/airbyte-lib/docs/generated/airbyte_lib/caches.html +++ b/airbyte-lib/docs/generated/airbyte_lib/caches.html @@ -679,7 +679,7 @@
Inherited Members

For now, only one source at a time is supported. If this method is called multiple times, the last call will overwrite the previous one.

-

TODO: Expand this to handle mutliple sources.

+

TODO: Expand this to handle multiple sources.

diff --git a/airbyte-lib/examples/run_faker.py b/airbyte-lib/examples/run_faker.py new file mode 100644 index 000000000000..3583a370000d --- /dev/null +++ b/airbyte-lib/examples/run_faker.py @@ -0,0 +1,31 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +"""A simple test of AirbyteLib, using the Faker source connector. + +Usage (from airbyte-lib root directory): +> poetry run python ./examples/run_faker.py + +No setup is needed, but you may need to delete the .venv-source-faker folder +if your installation gets interrupted or corrupted. +""" +from __future__ import annotations + +import airbyte_lib as ab + + +SCALE = 1_000_000 # Number of records to generate between users and purchases. + + +source = ab.get_connector( + "source-faker", + pip_url="-e ../airbyte-integrations/connectors/source-faker", + config={"count": SCALE / 2}, + install_if_missing=True, +) +source.check() +source.set_streams(["products", "users", "purchases"]) + +cache = ab.new_local_cache() +result = source.read(cache) + +for name, records in result.cache.streams.items(): + print(f"Stream {name}: {len(list(records))} records") diff --git a/airbyte-lib/poetry.lock b/airbyte-lib/poetry.lock index 5dafb19e798a..5ac62115d29d 100644 --- a/airbyte-lib/poetry.lock +++ b/airbyte-lib/poetry.lock @@ -525,6 +525,20 @@ docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1 testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] typing = ["typing-extensions (>=4.8)"] +[[package]] +name = "freezegun" +version = "1.4.0" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.7" +files = [ + {file = "freezegun-1.4.0-py3-none-any.whl", hash = "sha256:55e0fc3c84ebf0a96a5aa23ff8b53d70246479e9a68863f1fcac5a3e52f19dd6"}, + {file = "freezegun-1.4.0.tar.gz", hash = "sha256:10939b0ba0ff5adaecf3b06a5c2f73071d9678e507c5eaedb23c761d56ac774b"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + [[package]] name = "genson" version = "1.2.2" @@ -877,6 +891,30 @@ six = ">=1.11.0" format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] +[[package]] +name = "markdown-it-py" +version = "3.0.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +optional = false +python-versions = ">=3.8" +files = [ + {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, + {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + [[package]] name = "markupsafe" version = "2.1.4" @@ -946,6 +984,17 @@ files = [ {file = "MarkupSafe-2.1.4.tar.gz", hash = "sha256:3aae9af4cac263007fd6309c64c6ab4506dd2b79382d9d19a1994f9240b8db4f"}, ] +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] + [[package]] name = "mypy" version = "1.8.0" @@ -1915,6 +1964,24 @@ redis = ["redis (>=3)"] security = ["itsdangerous (>=2.0)"] yaml = ["pyyaml (>=5.4)"] +[[package]] +name = "rich" +version = "13.7.0" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "rich-13.7.0-py3-none-any.whl", hash = "sha256:6da14c108c4866ee9520bbffa71f6fe3962e193b7da68720583850cd4548e235"}, + {file = "rich-13.7.0.tar.gz", hash = "sha256:5cb5123b5cf9ee70584244246816e9114227e0b98ad9176eede6ad54bf5403fa"}, +] + +[package.dependencies] +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<9)"] + [[package]] name = "rpds-py" version = "0.17.1" @@ -2494,4 +2561,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "5eba75179b62f56be141db82121ca9e1c623944306172d7bdacaf5388e6f3384" +content-hash = "7ed61ca7eaed73dbd7e0800aa87fcd5b2583739048d53e9e564fe2a6defa483f" diff --git a/airbyte-lib/pyproject.toml b/airbyte-lib/pyproject.toml index a4749bbd750b..4625e688dc41 100644 --- a/airbyte-lib/pyproject.toml +++ b/airbyte-lib/pyproject.toml @@ -28,6 +28,7 @@ pyarrow = "^14.0.2" # Psycopg3 is not supported in SQLAlchemy 1.x: # psycopg = {extras = ["binary", "pool"], version = "^3.1.16"} +rich = "^13.7.0" [tool.poetry.group.dev.dependencies] @@ -44,6 +45,7 @@ ruff = "^0.1.11" types-jsonschema = "^4.20.0.0" google-cloud-secret-manager = "^2.17.0" types-requests = "2.31.0.4" +freezegun = "^1.4.0" [build-system] requires = ["poetry-core"] diff --git a/airbyte-lib/tests/conftest.py b/airbyte-lib/tests/conftest.py index caabe34cfed5..17009133a89d 100644 --- a/airbyte-lib/tests/conftest.py +++ b/airbyte-lib/tests/conftest.py @@ -12,6 +12,7 @@ import docker import psycopg2 as psycopg import pytest +from _pytest.nodes import Item from google.cloud import secretmanager from pytest_docker.plugin import get_docker_ip @@ -25,6 +26,32 @@ PYTEST_POSTGRES_PORT = 5432 +def pytest_collection_modifyitems(items: list[Item]) -> None: + """Override default pytest behavior, sorting our tests in a sensible execution order. + + In general, we want faster tests to run first, so that we can get feedback faster. + + Running lint tests first is helpful because they are fast and can catch typos and other errors. + + Otherwise tests are run based on an alpha-based natural sort, where 'unit' tests run after + 'integration' tests because 'u' comes after 'i' alphabetically. + """ + def test_priority(item: Item) -> int: + if 'lint_tests' in str(item.fspath): + return 1 # lint tests have high priority + elif 'unit_tests' in str(item.fspath): + return 2 # unit tests have highest priority + elif 'docs_tests' in str(item.fspath): + return 3 # doc tests have medium priority + elif 'integration_tests' in str(item.fspath): + return 4 # integration tests have the lowest priority + else: + return 5 # all other tests have lower priority + + # Sort the items list in-place based on the test_priority function + items.sort(key=test_priority) + + def is_port_in_use(port): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: return s.connect_ex(("localhost", port)) == 0 @@ -128,6 +155,7 @@ def new_pg_cache_config(pg_dsn): ) yield config + @pytest.fixture def snowflake_config(): if "GCP_GSM_CREDENTIALS" not in os.environ: diff --git a/airbyte-lib/tests/unit_tests/test_progress.py b/airbyte-lib/tests/unit_tests/test_progress.py new file mode 100644 index 000000000000..377df860bb57 --- /dev/null +++ b/airbyte-lib/tests/unit_tests/test_progress.py @@ -0,0 +1,174 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import datetime +from textwrap import dedent +import time +import pytest +from freezegun import freeze_time +from airbyte_lib.progress import ReadProgress, _get_elapsed_time_str, _to_time_str +from dateutil.tz import tzlocal + +# Calculate the offset from UTC in hours +tz_offset_hrs = int(datetime.datetime.now(tzlocal()).utcoffset().total_seconds() / 3600) + + +@freeze_time("2022-01-01") +def test_read_progress_initialization(): + progress = ReadProgress() + assert progress.num_streams_expected == 0 + assert progress.read_start_time == 1640995200.0 # Unix timestamp for 2022-01-01 + assert progress.total_records_read == 0 + assert progress.total_records_written == 0 + assert progress.total_batches_written == 0 + assert progress.written_stream_names == set() + assert progress.finalize_start_time is None + assert progress.finalize_end_time is None + assert progress.total_records_finalized == 0 + assert progress.total_batches_finalized == 0 + assert progress.finalized_stream_names == set() + assert progress.last_update_time is None + + +@freeze_time("2022-01-01") +def test_read_progress_reset(): + progress = ReadProgress() + progress.reset(5) + assert progress.num_streams_expected == 5 + assert progress.read_start_time == 1640995200.0 + assert progress.total_records_read == 0 + assert progress.total_records_written == 0 + assert progress.total_batches_written == 0 + assert progress.written_stream_names == set() + assert progress.finalize_start_time is None + assert progress.finalize_end_time is None + assert progress.total_records_finalized == 0 + assert progress.total_batches_finalized == 0 + assert progress.finalized_stream_names == set() + +@freeze_time("2022-01-01") +def test_read_progress_log_records_read(): + progress = ReadProgress() + progress.log_records_read(100) + assert progress.total_records_read == 100 + +@freeze_time("2022-01-01") +def test_read_progress_log_batch_written(): + progress = ReadProgress() + progress.log_batch_written("stream1", 50) + assert progress.total_records_written == 50 + assert progress.total_batches_written == 1 + assert progress.written_stream_names == {"stream1"} + +@freeze_time("2022-01-01") +def test_read_progress_log_batches_finalizing(): + progress = ReadProgress() + progress.log_batches_finalizing("stream1", 1) + assert progress.finalize_start_time == 1640995200.0 + +@freeze_time("2022-01-01") +def test_read_progress_log_batches_finalized(): + progress = ReadProgress() + progress.log_batches_finalized("stream1", 1) + assert progress.total_batches_finalized == 1 + +@freeze_time("2022-01-01") +def test_read_progress_log_stream_finalized(): + progress = ReadProgress() + progress.log_stream_finalized("stream1") + assert progress.finalized_stream_names == {"stream1"} + + +def test_get_elapsed_time_str(): + assert _get_elapsed_time_str(30) == "30 seconds" + assert _get_elapsed_time_str(90) == "1min 30s" + assert _get_elapsed_time_str(600) == "10min" + assert _get_elapsed_time_str(3600) == "1hr 0min" + + +@freeze_time("2022-01-01 0:00:00") +def test_get_time_str(): + assert _to_time_str(time.time()) == "00:00:00" + + +def _assert_lines(expected_lines, actual_lines: list[str] | str): + if isinstance(actual_lines, list): + actual_lines = "\n".join(actual_lines) + for line in expected_lines: + assert line in actual_lines, f"Missing line: {line}" + +def test_get_status_message_after_finalizing_records(): + + # Test that we can render the initial status message before starting to read + with freeze_time("2022-01-01 00:00:00"): + progress = ReadProgress() + expected_lines = [ + "Started reading at 00:00:00.", + "Read **0** records over **0 seconds** (0.0 records / second).", + ] + _assert_lines(expected_lines, progress._get_status_message()) + + # Test after reading some records + with freeze_time("2022-01-01 00:01:00"): + progress.log_records_read(100) + expected_lines = [ + "Started reading at 00:00:00.", + "Read **100** records over **60 seconds** (1.7 records / second).", + ] + _assert_lines(expected_lines, progress._get_status_message()) + + # Advance the day and reset the progress + with freeze_time("2022-01-02 00:00:00"): + progress = ReadProgress() + progress.reset(1) + expected_lines = [ + "Started reading at 00:00:00.", + "Read **0** records over **0 seconds** (0.0 records / second).", + ] + _assert_lines(expected_lines, progress._get_status_message()) + + # Test after writing some records and starting to finalize + with freeze_time("2022-01-02 00:01:00"): + progress.log_records_read(100) + progress.log_batch_written("stream1", 50) + progress.log_batches_finalizing("stream1", 1) + expected_lines = [ + "## Read Progress", + "Started reading at 00:00:00.", + "Read **100** records over **60 seconds** (1.7 records / second).", + "Wrote **50** records over 1 batches.", + "Finished reading at 00:01:00.", + "Started finalizing streams at 00:01:00.", + ] + _assert_lines(expected_lines, progress._get_status_message()) + + # Test after finalizing some records + with freeze_time("2022-01-02 00:02:00"): + progress.log_batches_finalized("stream1", 1) + expected_lines = [ + "## Read Progress", + "Started reading at 00:00:00.", + "Read **100** records over **60 seconds** (1.7 records / second).", + "Wrote **50** records over 1 batches.", + "Finished reading at 00:01:00.", + "Started finalizing streams at 00:01:00.", + "Finalized **1** batches over 60 seconds.", + ] + _assert_lines(expected_lines, progress._get_status_message()) + + # Test after finalizing all records + with freeze_time("2022-01-02 00:02:00"): + progress.log_stream_finalized("stream1") + message = progress._get_status_message() + expected_lines = [ + "## Read Progress", + "Started reading at 00:00:00.", + "Read **100** records over **60 seconds** (1.7 records / second).", + "Wrote **50** records over 1 batches.", + "Finished reading at 00:01:00.", + "Started finalizing streams at 00:01:00.", + "Finalized **1** batches over 60 seconds.", + "Completed 1 out of 1 streams:", + "- stream1", + "Total time elapsed: 2min 0s", + ] + _assert_lines(expected_lines, message) From a4738b93da0215a95a92110b03840ee88158a815 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Mon, 29 Jan 2024 22:56:24 -0800 Subject: [PATCH 11/96] AirbyteLib: DuckDB Perf Boost (#34589) --- airbyte-lib/airbyte_lib/caches/duckdb.py | 24 ++++++++++++++++++++++-- airbyte-lib/examples/run_faker.py | 2 +- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/airbyte-lib/airbyte_lib/caches/duckdb.py b/airbyte-lib/airbyte_lib/caches/duckdb.py index 0cdb8b28c99b..76f79968a503 100644 --- a/airbyte-lib/airbyte_lib/caches/duckdb.py +++ b/airbyte-lib/airbyte_lib/caches/duckdb.py @@ -5,6 +5,7 @@ from __future__ import annotations from pathlib import Path +from textwrap import dedent from typing import cast from overrides import overrides @@ -158,6 +159,25 @@ def _write_files_to_new_table( ) -> str: """Write a file(s) to a new table. - TODO: Optimize this for DuckDB instead of calling the base implementation. + We use DuckDB's `read_parquet` function to efficiently read the files and insert + them into the table in a single operation. + + Note: This implementation is fragile in regards to column ordering. However, since + we are inserting into a temp table we have just created, there should be no + drift between the table schema and the file schema. """ - return super()._write_files_to_new_table(files, stream_name, batch_id) + temp_table_name = self._create_table_for_loading( + stream_name=stream_name, + batch_id=batch_id, + ) + files_list = ", ".join([f"'{f!s}'" for f in files]) + insert_statement = dedent( + f""" + INSERT INTO {self.config.schema_name}.{temp_table_name} + SELECT * FROM read_parquet( + [{files_list}] + ) + """ + ) + self._execute_sql(insert_statement) + return temp_table_name diff --git a/airbyte-lib/examples/run_faker.py b/airbyte-lib/examples/run_faker.py index 3583a370000d..c9f0db1e6ba0 100644 --- a/airbyte-lib/examples/run_faker.py +++ b/airbyte-lib/examples/run_faker.py @@ -12,7 +12,7 @@ import airbyte_lib as ab -SCALE = 1_000_000 # Number of records to generate between users and purchases. +SCALE = 5_000_000 # Number of records to generate between users and purchases. source = ab.get_connector( From 923fcc50c2a1c0840699fe538c9c09e61a2aea23 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 30 Jan 2024 09:49:50 +0100 Subject: [PATCH 12/96] airbyte-ci: Switch to prod pypi (#34606) --- .github/actions/run-airbyte-ci/action.yml | 5 ++ .../publish-airbyte-lib-command-manually.yml | 4 +- airbyte-ci/connectors/pipelines/README.md | 47 ++++++++++++------- .../airbyte_ci/connectors/publish/commands.py | 19 +++++++- .../airbyte_ci/connectors/publish/context.py | 4 ++ .../airbyte_ci/connectors/publish/pipeline.py | 10 ++++ .../airbyte_ci/poetry/publish/commands.py | 7 +-- .../airbyte_ci/steps/python_registry.py | 8 +++- .../connectors/pipelines/pipelines/consts.py | 2 +- .../contexts/python_registry_publish.py | 5 +- .../connectors/pipelines/pyproject.toml | 2 +- .../pipelines/tests/test_publish.py | 22 +++++---- 12 files changed, 95 insertions(+), 40 deletions(-) diff --git a/.github/actions/run-airbyte-ci/action.yml b/.github/actions/run-airbyte-ci/action.yml index 45e8262a5837..753c041727e7 100644 --- a/.github/actions/run-airbyte-ci/action.yml +++ b/.github/actions/run-airbyte-ci/action.yml @@ -79,6 +79,10 @@ inputs: python_registry_token: description: "Python registry API token to publish python package" required: false + python_registry_url: + description: "Python registry URL to publish python package" + default: "https://upload.pypi.org/legacy/" + required: false runs: using: "composite" @@ -135,6 +139,7 @@ runs: PRODUCTION: ${{ inputs.production }} PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} PYTHON_REGISTRY_TOKEN: ${{ inputs.python_registry_token }} + PYTHON_REGISTRY_URL: ${{ inputs.python_registry_url }} S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ inputs.s3_build_cache_access_key_id }} S3_BUILD_CACHE_SECRET_KEY: ${{ inputs.s3_build_cache_secret_key }} SENTRY_DSN: ${{ inputs.sentry_dsn }} diff --git a/.github/workflows/publish-airbyte-lib-command-manually.yml b/.github/workflows/publish-airbyte-lib-command-manually.yml index a4fbfbc81495..e596444414d3 100644 --- a/.github/workflows/publish-airbyte-lib-command-manually.yml +++ b/.github/workflows/publish-airbyte-lib-command-manually.yml @@ -37,7 +37,7 @@ jobs: uses: actions/checkout@v3 - name: Publish id: publish-airbyte-lib - uses: ./.github/actions/run-dagger-pipeline + uses: ./.github/actions/run-airbyte-ci with: context: "manual" dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} @@ -53,5 +53,5 @@ jobs: s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} - subcommand: 'poetry --package-path=airbyte-lib publish --registry-url="https://test.pypi.org/legacy/"' + subcommand: "poetry --package-path=airbyte-lib publish" python_registry_token: ${{ secrets.PYPI_TOKEN }} diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 263333e3f15e..3e8ccd074580 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -385,16 +385,18 @@ Publish all connectors modified in the head commit: `airbyte-ci connectors --mod ### Options -| Option | Required | Default | Mapped environment variable | Description | -| ------------------------------------ | -------- | --------------- | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--pre-release/--main-release` | False | `--pre-release` | | Whether to publish the pre-release or the main release version of a connector. Defaults to pre-release. For main release you have to set the credentials to interact with the GCS bucket. | -| `--spec-cache-gcs-credentials` | False | | `SPEC_CACHE_GCS_CREDENTIALS` | The service account key to upload files to the GCS bucket hosting spec cache. | -| `--spec-cache-bucket-name` | False | | `SPEC_CACHE_BUCKET_NAME` | The name of the GCS bucket where specs will be cached. | -| `--metadata-service-gcs-credentials` | False | | `METADATA_SERVICE_GCS_CREDENTIALS` | The service account key to upload files to the GCS bucket hosting the metadata files. | -| `--metadata-service-bucket-name` | False | | `METADATA_SERVICE_BUCKET_NAME` | The name of the GCS bucket where metadata files will be uploaded. | -| `--slack-webhook` | False | | `SLACK_WEBHOOK` | The Slack webhook URL to send notifications to. | -| `--slack-channel` | False | | `SLACK_CHANNEL` | The Slack channel name to send notifications to. | -| `--ci-requirements` | False | | | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. | +| Option | Required | Default | Mapped environment variable | Description | +| ------------------------------------ | -------- | ------------------------------- | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--pre-release/--main-release` | False | `--pre-release` | | Whether to publish the pre-release or the main release version of a connector. Defaults to pre-release. For main release you have to set the credentials to interact with the GCS bucket. | +| `--spec-cache-gcs-credentials` | False | | `SPEC_CACHE_GCS_CREDENTIALS` | The service account key to upload files to the GCS bucket hosting spec cache. | +| `--spec-cache-bucket-name` | False | | `SPEC_CACHE_BUCKET_NAME` | The name of the GCS bucket where specs will be cached. | +| `--metadata-service-gcs-credentials` | False | | `METADATA_SERVICE_GCS_CREDENTIALS` | The service account key to upload files to the GCS bucket hosting the metadata files. | +| `--metadata-service-bucket-name` | False | | `METADATA_SERVICE_BUCKET_NAME` | The name of the GCS bucket where metadata files will be uploaded. | +| `--slack-webhook` | False | | `SLACK_WEBHOOK` | The Slack webhook URL to send notifications to. | +| `--slack-channel` | False | | `SLACK_CHANNEL` | The Slack channel name to send notifications to. | +| `--ci-requirements` | False | | | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. | +| `--python-registry-token` | False | | `PYTHON_REGISTRY_TOKEN` | The API token to authenticate with the registry. For pypi, the `pypi-` prefix needs to be specified | +| `--python-registry-url` | False | https://upload.pypi.org/legacy/ | `PYTHON_REGISTRY_URL` | The python registry to publish to. Defaults to main pypi | I've added an empty "Default" column, and you can fill in the default values as needed. @@ -406,14 +408,24 @@ flowchart TD validate[Validate the metadata file] check[Check if the connector image already exists] build[Build the connector image for all platform variants] + publish_to_python_registry[Push the connector image to the python registry if enabled] upload_spec[Upload connector spec to the spec cache bucket] push[Push the connector image from DockerHub, with platform variants] pull[Pull the connector image from DockerHub to check SPEC can be run and the image layers are healthy] upload_metadata[Upload its metadata file to the metadata service bucket] - validate-->check-->build-->upload_spec-->push-->pull-->upload_metadata + validate-->check-->build-->upload_spec-->publish_to_python_registry-->push-->pull-->upload_metadata ``` +#### Python registry publishing + +If `remoteRegistries.pypi.enabled` in the connector metadata is set to `true`, the connector will be published to the python registry. +To do so, the `--python-registry-token` and `--python-registry-url` options are used to authenticate with the registry and publish the connector. +If the current version of the connector is already published to the registry, the publish will be skipped. + +On a pre-release, the connector will be published as a `.dev` version. + + ### `connectors bump_version` command Bump the version of the selected connectors. @@ -534,12 +546,12 @@ For poetry packages, the package name and version can be taken from the `pyproje #### Options -| Option | Required | Default | Mapped environment variable | Description | -| ------------------------- | -------- | ----------------------- | --------------------------- | -------------------------------------------------------------------------------------------------------- | -| `--publish-name` | False | | | The name of the package. Not required for poetry packages that define it in the `pyproject.toml` file | -| `--publish-version` | False | | | The version of the package. Not required for poetry packages that define it in the `pyproject.toml` file | -| `--python-registry-token` | True | | PYTHON_REGISTRY_TOKEN | The API token to authenticate with the registry. For pypi, the `pypi-` prefix needs to be specified | -| `--registry-url` | False | https://pypi.org/simple | | The python registry to publish to. Defaults to main pypi | +| Option | Required | Default | Mapped environment variable | Description | +| ------------------------- | -------- | ------------------------------- | --------------------------- | -------------------------------------------------------------------------------------------------------- | +| `--publish-name` | False | | | The name of the package. Not required for poetry packages that define it in the `pyproject.toml` file | +| `--publish-version` | False | | | The version of the package. Not required for poetry packages that define it in the `pyproject.toml` file | +| `--python-registry-token` | True | | PYTHON_REGISTRY_TOKEN | The API token to authenticate with the registry. For pypi, the `pypi-` prefix needs to be specified | +| `--python-registry-url` | False | https://upload.pypi.org/legacy/ | PYTHON_REGISTRY_URL | The python registry to publish to. Defaults to main pypi | ### `metadata` command subgroup @@ -597,6 +609,7 @@ E.G.: running `pytest` on a specific test folder: | Version | PR | Description | | ------- | ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| 3.9.0 | [#34606](https://github.com/airbytehq/airbyte/pull/34606) | Allow configuration of python registry URL via environment variable. | | 3.8.1 | [#34607](https://github.com/airbytehq/airbyte/pull/34607) | Improve gradle dependency cache volume protection. | | 3.8.0 | [#34316](https://github.com/airbytehq/airbyte/pull/34316) | Expose Dagger engine image name in `--ci-requirements` and add `--ci-requirements` to the `airbyte-ci` root command group. | | 3.7.3 | [#34560](https://github.com/airbytehq/airbyte/pull/34560) | Simplify Gradle task execution framework by removing local maven repo support. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py index 0de1d7a2032f..c5fbfc414486 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py @@ -11,7 +11,7 @@ from pipelines.cli.click_decorators import click_ci_requirements_option from pipelines.cli.confirm_prompt import confirm from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand -from pipelines.consts import ContextState +from pipelines.consts import DEFAULT_PYTHON_PACKAGE_REGISTRY_URL, ContextState from pipelines.helpers.utils import fail_if_missing_docker_hub_creds @@ -59,6 +59,19 @@ envvar="SLACK_CHANNEL", default="#connector-publish-updates", ) +@click.option( + "--python-registry-token", + help="Access token for python registry", + type=click.STRING, + envvar="PYTHON_REGISTRY_TOKEN", +) +@click.option( + "--python-registry-url", + help="Which python registry registry to publish to. If not set, the default pypi is used. For test pypi, use https://test.pypi.org/legacy/", + type=click.STRING, + default=DEFAULT_PYTHON_PACKAGE_REGISTRY_URL, + envvar="PYTHON_REGISTRY_URL", +) @click.pass_context async def publish( ctx: click.Context, @@ -69,6 +82,8 @@ async def publish( metadata_service_gcs_credentials: str, slack_webhook: str, slack_channel: str, + python_registry_token: str, + python_registry_url: str, ) -> bool: ctx.obj["spec_cache_gcs_credentials"] = spec_cache_gcs_credentials ctx.obj["spec_cache_bucket_name"] = spec_cache_bucket_name @@ -109,6 +124,8 @@ async def publish( s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"), s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"), use_local_cdk=ctx.obj.get("use_local_cdk"), + python_registry_token=python_registry_token, + python_registry_url=python_registry_url, ) for connector in ctx.obj["selected_connectors_with_modified_files"] ] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py index 829ab07b4e0a..55136809d8ce 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py @@ -46,12 +46,16 @@ def __init__( s3_build_cache_access_key_id: Optional[str] = None, s3_build_cache_secret_key: Optional[str] = None, use_local_cdk: bool = False, + python_registry_token: Optional[str] = None, + python_registry_url: Optional[str] = None, ) -> None: self.pre_release = pre_release self.spec_cache_bucket_name = spec_cache_bucket_name self.metadata_bucket_name = metadata_bucket_name self.spec_cache_gcs_credentials = sanitize_gcs_credentials(spec_cache_gcs_credentials) self.metadata_service_gcs_credentials = sanitize_gcs_credentials(metadata_service_gcs_credentials) + self.python_registry_token = python_registry_token + self.python_registry_url = python_registry_url pipeline_name = f"Publish {connector.technical_name}" pipeline_name = pipeline_name + " (pre-release)" if pre_release else pipeline_name diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index c6e4002829c3..ac7514439cb0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -352,6 +352,16 @@ async def _run_python_registry_publish_pipeline(context: PublishConnectorContext if not python_registry_context: return results, False + if not context.python_registry_token or not context.python_registry_url: + # If the python registry token or url are not set, we can't publish to the python registry - stop the pipeline. + return [ + StepResult( + PublishToPythonRegistry(python_registry_context), + status=StepStatus.FAILURE, + stderr="Pypi publishing is enabled, but python registry token or url are not set.", + ) + ], True + check_python_registry_package_exists_results = await CheckPythonRegistryPackageDoesNotExist(python_registry_context).run() results.append(check_python_registry_package_exists_results) if check_python_registry_package_exists_results.status is StepStatus.SKIPPED: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py index 29785f831266..16bded0b0c34 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py @@ -47,10 +47,11 @@ def _validate_python_version(_ctx: dict, _param: dict, value: Optional[str]) -> envvar="PYTHON_REGISTRY_TOKEN", ) @click.option( - "--registry-url", + "--python-registry-url", help="Which registry to publish to. If not set, the default pypi is used. For test pypi, use https://test.pypi.org/legacy/", type=click.STRING, default=DEFAULT_PYTHON_PACKAGE_REGISTRY_URL, + envvar="PYTHON_REGISTRY_URL", ) @click.option( "--publish-name", @@ -69,7 +70,7 @@ async def publish( ctx: click.Context, click_pipeline_context: ClickPipelineContext, python_registry_token: str, - registry_url: str, + python_registry_url: str, publish_name: Optional[str], publish_version: Optional[str], ) -> bool: @@ -85,7 +86,7 @@ async def publish( ci_context=ctx.obj.get("ci_context"), ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], python_registry_token=python_registry_token, - registry=registry_url, + registry=python_registry_url, package_path=ctx.obj["package_path"], package_name=publish_name, version=publish_version, diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py index aec2e30bb3da..2bfebec127b5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py @@ -116,9 +116,11 @@ async def _poetry_publish(self, package_dir_to_publish: Directory) -> StepResult .with_directory("package", package_dir_to_publish) .with_workdir("package") .with_new_file(PYPROJECT_TOML_FILE_PATH, contents=tomli_w.dumps(contents)) + # Make sure these steps are always executed and not cached as they are triggering a side-effect (calling the registry) + # Env var setting needs to be in this block as well to make sure a change of the env var will be propagated correctly + .with_env_variable("CACHEBUSTER", str(uuid.uuid4())) .with_exec(["poetry", "config", "repositories.mypypi", self.context.registry]) .with_exec(sh_dash_c(["poetry config pypi-token.mypypi $PYTHON_REGISTRY_TOKEN"])) - .with_env_variable("CACHEBUSTER", str(uuid.uuid4())) .with_exec(sh_dash_c(["poetry publish --build --repository mypypi -vvv --no-interaction"])) ) @@ -156,9 +158,11 @@ async def _pip_publish(self, package_dir_to_publish: Directory) -> StepResult: .with_new_file("setup.cfg", contents=setup_cfg) .with_exec(["pip", "install", "--upgrade", "setuptools", "wheel"]) .with_exec(["python", SETUP_PY_FILE_PATH, "sdist", "bdist_wheel"]) + # Make sure these steps are always executed and not cached as they are triggering a side-effect (calling the registry) + # Env var setting needs to be in this block as well to make sure a change of the env var will be propagated correctly + .with_env_variable("CACHEBUSTER", str(uuid.uuid4())) .with_secret_variable("TWINE_USERNAME", pypi_username) .with_secret_variable("TWINE_PASSWORD", pypi_password) - .with_env_variable("CACHEBUSTER", str(uuid.uuid4())) .with_exec(["twine", "upload", "--verbose", "--repository-url", self.context.registry, "dist/*"]) ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/consts.py b/airbyte-ci/connectors/pipelines/pipelines/consts.py index 203b320bb6fc..966c484b15f1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/consts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/consts.py @@ -60,7 +60,7 @@ POETRY_CACHE_PATH = "/root/.cache/pypoetry" STORAGE_DRIVER = "fuse-overlayfs" SETUP_PY_FILE_PATH = "setup.py" -DEFAULT_PYTHON_PACKAGE_REGISTRY_URL = "https://pypi.org/simple" +DEFAULT_PYTHON_PACKAGE_REGISTRY_URL = "https://upload.pypi.org/legacy/" class CIContext(str, Enum): diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/python_registry_publish.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/python_registry_publish.py index ee45760d2f9d..a189b476b892 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/python_registry_publish.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/python_registry_publish.py @@ -2,7 +2,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import os from dataclasses import dataclass from datetime import datetime from typing import Optional, Type @@ -86,8 +85,8 @@ async def from_publish_connector_context( version = f"{version}.dev{release_candidate_tag}" pypi_context = cls( - python_registry_token=os.environ["PYTHON_REGISTRY_TOKEN"], - registry="https://test.pypi.org/legacy/", # TODO: go live + python_registry_token=str(connector_context.python_registry_token), + registry=str(connector_context.python_registry_url), package_path=str(connector_context.connector.code_directory), package_name=current_metadata["remoteRegistries"]["pypi"]["packageName"], version=version, diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index d3dca0a54f60..8828208d99dd 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "3.8.1" +version = "3.9.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] diff --git a/airbyte-ci/connectors/pipelines/tests/test_publish.py b/airbyte-ci/connectors/pipelines/tests/test_publish.py index ce8913e648ab..471c89ae6ece 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_publish.py +++ b/airbyte-ci/connectors/pipelines/tests/test_publish.py @@ -5,7 +5,6 @@ import os import random from typing import List -from unittest.mock import patch import anyio import pytest @@ -339,13 +338,14 @@ async def test_run_connector_publish_pipeline_when_image_does_not_exist( @pytest.mark.parametrize( - "pypi_enabled, pypi_package_does_not_exist_status, publish_step_status, expect_publish_to_pypi_called, expect_build_connector_called", + "pypi_enabled, pypi_package_does_not_exist_status, publish_step_status, expect_publish_to_pypi_called, expect_build_connector_called,api_token", [ - pytest.param(True, StepStatus.SUCCESS, StepStatus.SUCCESS, True, True, id="happy_path"), - pytest.param(False, StepStatus.SUCCESS, StepStatus.SUCCESS, False, True, id="pypi_disabled, skip all pypi steps"), - pytest.param(True, StepStatus.SKIPPED, StepStatus.SUCCESS, False, True, id="pypi_package_exists, skip publish_to_pypi"), - pytest.param(True, StepStatus.SUCCESS, StepStatus.FAILURE, True, False, id="publish_step_fails, abort"), - pytest.param(True, StepStatus.FAILURE, StepStatus.FAILURE, False, False, id="pypi_package_does_not_exist_fails, abort"), + pytest.param(True, StepStatus.SUCCESS, StepStatus.SUCCESS, True, True, "test", id="happy_path"), + pytest.param(False, StepStatus.SUCCESS, StepStatus.SUCCESS, False, True, "test", id="pypi_disabled, skip all pypi steps"), + pytest.param(True, StepStatus.SKIPPED, StepStatus.SUCCESS, False, True, "test", id="pypi_package_exists, skip publish_to_pypi"), + pytest.param(True, StepStatus.SUCCESS, StepStatus.FAILURE, True, False, "test", id="publish_step_fails, abort"), + pytest.param(True, StepStatus.FAILURE, StepStatus.FAILURE, False, False, "test", id="pypi_package_does_not_exist_fails, abort"), + pytest.param(True, StepStatus.SUCCESS, StepStatus.SUCCESS, False, False, None, id="no_api_token, abort"), ], ) async def test_run_connector_python_registry_publish_pipeline( @@ -355,6 +355,7 @@ async def test_run_connector_python_registry_publish_pipeline( publish_step_status, expect_publish_to_pypi_called, expect_build_connector_called, + api_token, ): for module, to_mock in STEPS_TO_PATCH: @@ -389,14 +390,15 @@ async def test_run_connector_python_registry_publish_pipeline( code_directory="path/to/connector", metadata={"dockerImageTag": "1.2.3", "remoteRegistries": {"pypi": {"enabled": pypi_enabled, "packageName": "test"}}}, ), + python_registry_token=api_token, + python_registry_url="https://test.pypi.org/legacy/", ) semaphore = anyio.Semaphore(1) - with patch.dict(os.environ, {"PYTHON_REGISTRY_TOKEN": "test"}): - await publish_pipeline.run_connector_publish_pipeline(context, semaphore) + await publish_pipeline.run_connector_publish_pipeline(context, semaphore) if expect_publish_to_pypi_called: mocked_publish_to_python_registry.return_value.run.assert_called_once() # assert that the first argument passed to mocked_publish_to_pypi contains the things from the context - assert mocked_publish_to_python_registry.call_args.args[0].python_registry_token == "test" + assert mocked_publish_to_python_registry.call_args.args[0].python_registry_token == api_token assert mocked_publish_to_python_registry.call_args.args[0].package_metadata.name == "test" assert mocked_publish_to_python_registry.call_args.args[0].package_metadata.version == "1.2.3" assert mocked_publish_to_python_registry.call_args.args[0].registry == "https://test.pypi.org/legacy/" From 8f727208ee637c7335faccc0515679cab48bf242 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 30 Jan 2024 10:22:13 +0100 Subject: [PATCH 13/96] airbyte-lib: Refactor connectors (#34552) --- .../connectors/source-activecampaign/main.py | 9 ++--- .../connectors/source-activecampaign/setup.py | 19 +++++++++- .../source_activecampaign/run.py | 14 ++++++++ .../connectors/source-adjust/main.py | 9 ++--- .../connectors/source-adjust/setup.py | 19 +++++++++- .../source-adjust/source_adjust/run.py | 14 ++++++++ .../connectors/source-aha/main.py | 9 ++--- .../connectors/source-aha/setup.py | 19 +++++++++- .../connectors/source-aha/source_aha/run.py | 14 ++++++++ .../connectors/source-aircall/main.py | 9 ++--- .../connectors/source-aircall/setup.py | 19 +++++++++- .../source-aircall/source_aircall/run.py | 14 ++++++++ .../connectors/source-airtable/main.py | 9 ++--- .../connectors/source-airtable/setup.py | 5 +++ .../source-airtable/source_airtable/run.py | 14 ++++++++ .../connectors/source-alpha-vantage/main.py | 9 ++--- .../connectors/source-alpha-vantage/setup.py | 19 +++++++++- .../source_alpha_vantage/run.py | 14 ++++++++ .../connectors/source-amazon-ads/main.py | 11 ++---- .../connectors/source-amazon-ads/setup.py | 5 +++ .../source_amazon_ads/run.py | 16 +++++++++ .../source-amazon-seller-partner/main.py | 12 ++----- .../source-amazon-seller-partner/setup.py | 5 +++ .../source_amazon_seller_partner/run.py | 17 +++++++++ .../connectors/source-amazon-sqs/main.py | 9 ++--- .../connectors/source-amazon-sqs/setup.py | 5 +++ .../source_amazon_sqs/run.py | 14 ++++++++ .../connectors/source-amplitude/main.py | 9 ++--- .../connectors/source-amplitude/setup.py | 19 +++++++++- .../source-amplitude/source_amplitude/run.py | 14 ++++++++ .../connectors/source-apify-dataset/setup.py | 14 +++++++- .../connectors/source-appfollow/main.py | 9 ++--- .../connectors/source-appfollow/setup.py | 19 +++++++++- .../source-appfollow/source_appfollow/run.py | 14 ++++++++ .../source-apple-search-ads/main.py | 9 ++--- .../source-apple-search-ads/setup.py | 19 +++++++++- .../source_apple_search_ads/run.py | 14 ++++++++ .../connectors/source-appsflyer/main.py | 9 ++--- .../connectors/source-appsflyer/setup.py | 5 +++ .../source-appsflyer/source_appsflyer/run.py | 14 ++++++++ .../connectors/source-appstore-singer/main.py | 9 ++--- .../source-appstore-singer/setup.py | 5 +++ .../source_appstore_singer/run.py | 14 ++++++++ .../connectors/source-asana/main.py | 9 ++--- .../connectors/source-asana/setup.py | 5 +++ .../source-asana/source_asana/run.py | 14 ++++++++ .../connectors/source-ashby/main.py | 9 ++--- .../connectors/source-ashby/setup.py | 19 +++++++++- .../source-ashby/source_ashby/run.py | 14 ++++++++ .../connectors/source-auth0/main.py | 9 ++--- .../connectors/source-auth0/setup.py | 19 +++++++++- .../source-auth0/source_auth0/run.py | 14 ++++++++ .../connectors/source-aws-cloudtrail/main.py | 9 ++--- .../connectors/source-aws-cloudtrail/setup.py | 5 +++ .../source_aws_cloudtrail/run.py | 14 ++++++++ .../source-azure-blob-storage/main.py | 29 ++------------- .../source-azure-blob-storage/setup.py | 5 +++ .../source_azure_blob_storage/run.py | 34 ++++++++++++++++++ .../connectors/source-azure-table/main.py | 9 ++--- .../connectors/source-azure-table/setup.py | 5 +++ .../source_azure_table/run.py | 14 ++++++++ .../connectors/source-babelforce/main.py | 9 ++--- .../connectors/source-babelforce/setup.py | 19 +++++++++- .../source_babelforce/run.py | 14 ++++++++ .../connectors/source-bamboo-hr/main.py | 9 ++--- .../connectors/source-bamboo-hr/setup.py | 5 +++ .../source-bamboo-hr/source_bamboo_hr/run.py | 14 ++++++++ .../connectors/source-bigcommerce/main.py | 9 ++--- .../connectors/source-bigcommerce/setup.py | 19 +++++++++- .../source_bigcommerce/run.py | 14 ++++++++ .../connectors/source-bing-ads/main.py | 9 ++--- .../connectors/source-bing-ads/setup.py | 5 +++ .../source-bing-ads/source_bing_ads/run.py | 14 ++++++++ .../connectors/source-braintree/main.py | 9 ++--- .../connectors/source-braintree/setup.py | 19 +++++++++- .../source-braintree/source_braintree/run.py | 14 ++++++++ .../connectors/source-braze/main.py | 9 ++--- .../connectors/source-braze/setup.py | 19 +++++++++- .../source-braze/source_braze/run.py | 14 ++++++++ .../connectors/source-breezometer/main.py | 9 ++--- .../connectors/source-breezometer/setup.py | 19 +++++++++- .../source_breezometer/run.py | 14 ++++++++ .../connectors/source-callrail/main.py | 9 ++--- .../connectors/source-callrail/setup.py | 19 +++++++++- .../source-callrail/source_callrail/run.py | 14 ++++++++ .../connectors/source-captain-data/main.py | 9 ++--- .../connectors/source-captain-data/setup.py | 19 +++++++++- .../source_captain_data/run.py | 14 ++++++++ .../connectors/source-cart/main.py | 9 ++--- .../connectors/source-cart/setup.py | 5 +++ .../connectors/source-cart/source_cart/run.py | 14 ++++++++ .../connectors/source-chargebee/main.py | 9 ++--- .../connectors/source-chargebee/setup.py | 19 +++++++++- .../source-chargebee/source_chargebee/run.py | 14 ++++++++ .../connectors/source-chargify/main.py | 9 ++--- .../connectors/source-chargify/setup.py | 19 +++++++++- .../source-chargify/source_chargify/run.py | 14 ++++++++ .../connectors/source-chartmogul/main.py | 9 ++--- .../connectors/source-chartmogul/setup.py | 5 +++ .../source_chartmogul/run.py | 14 ++++++++ .../connectors/source-clickup-api/main.py | 9 ++--- .../connectors/source-clickup-api/setup.py | 19 +++++++++- .../source_clickup_api/run.py | 14 ++++++++ .../connectors/source-clockify/main.py | 9 ++--- .../connectors/source-clockify/setup.py | 19 +++++++++- .../source-clockify/source_clockify/run.py | 14 ++++++++ .../connectors/source-close-com/main.py | 9 ++--- .../connectors/source-close-com/setup.py | 5 +++ .../source-close-com/source_close_com/run.py | 14 ++++++++ .../connectors/source-coda/main.py | 9 ++--- .../connectors/source-coda/setup.py | 19 +++++++++- .../connectors/source-coda/source_coda/run.py | 14 ++++++++ .../connectors/source-coin-api/main.py | 9 ++--- .../connectors/source-coin-api/setup.py | 19 +++++++++- .../source-coin-api/source_coin_api/run.py | 14 ++++++++ .../connectors/source-coingecko-coins/main.py | 9 ++--- .../source-coingecko-coins/setup.py | 19 +++++++++- .../source_coingecko_coins/run.py | 14 ++++++++ .../connectors/source-coinmarketcap/main.py | 9 ++--- .../connectors/source-coinmarketcap/setup.py | 19 +++++++++- .../source_coinmarketcap/run.py | 14 ++++++++ .../connectors/source-commcare/main.py | 9 ++--- .../connectors/source-commcare/setup.py | 19 +++++++++- .../source-commcare/source_commcare/run.py | 14 ++++++++ .../connectors/source-commercetools/main.py | 9 ++--- .../connectors/source-commercetools/setup.py | 19 +++++++++- .../source_commercetools/run.py | 14 ++++++++ .../connectors/source-configcat/main.py | 9 ++--- .../connectors/source-configcat/setup.py | 19 +++++++++- .../source-configcat/source_configcat/run.py | 14 ++++++++ .../connectors/source-confluence/main.py | 9 ++--- .../connectors/source-confluence/setup.py | 19 +++++++++- .../source_confluence/run.py | 14 ++++++++ .../connectors/source-convertkit/main.py | 9 ++--- .../connectors/source-convertkit/setup.py | 19 +++++++++- .../source_convertkit/run.py | 14 ++++++++ .../connectors/source-convex/main.py | 9 ++--- .../connectors/source-convex/setup.py | 19 +++++++++- .../source-convex/source_convex/run.py | 14 ++++++++ .../connectors/source-copper/main.py | 9 ++--- .../connectors/source-copper/setup.py | 19 +++++++++- .../source-copper/source_copper/run.py | 14 ++++++++ .../connectors/source-courier/main.py | 9 ++--- .../connectors/source-courier/setup.py | 19 +++++++++- .../source-courier/source_courier/run.py | 14 ++++++++ .../connectors/source-customer-io/main.py | 9 ++--- .../connectors/source-customer-io/setup.py | 19 +++++++++- .../source_customer_io/run.py | 14 ++++++++ .../connectors/source-datadog/main.py | 9 ++--- .../connectors/source-datadog/setup.py | 19 +++++++++- .../source-datadog/source_datadog/run.py | 14 ++++++++ .../connectors/source-datascope/main.py | 9 ++--- .../connectors/source-datascope/setup.py | 19 +++++++++- .../source-datascope/source_datascope/run.py | 14 ++++++++ .../connectors/source-delighted/main.py | 31 ++-------------- .../connectors/source-delighted/setup.py | 5 +++ .../source-delighted/source_delighted/run.py | 36 +++++++++++++++++++ .../connectors/source-dixa/main.py | 9 ++--- .../connectors/source-dixa/setup.py | 19 +++++++++- .../connectors/source-dixa/source_dixa/run.py | 14 ++++++++ .../connectors/source-dockerhub/main.py | 9 ++--- .../connectors/source-dockerhub/setup.py | 19 +++++++++- .../source-dockerhub/source_dockerhub/run.py | 14 ++++++++ .../connectors/source-dremio/main.py | 9 ++--- .../connectors/source-dremio/setup.py | 19 +++++++++- .../source-dremio/source_dremio/run.py | 14 ++++++++ .../connectors/source-drift/main.py | 9 ++--- .../connectors/source-drift/setup.py | 19 +++++++++- .../source-drift/source_drift/run.py | 14 ++++++++ .../connectors/source-dv-360/main.py | 9 ++--- .../connectors/source-dv-360/setup.py | 5 +++ .../source-dv-360/source_dv_360/run.py | 14 ++++++++ .../connectors/source-emailoctopus/main.py | 9 ++--- .../connectors/source-emailoctopus/setup.py | 19 +++++++++- .../source_emailoctopus/run.py | 14 ++++++++ .../connectors/source-everhour/main.py | 9 ++--- .../connectors/source-everhour/setup.py | 19 +++++++++- .../source-everhour/source_everhour/run.py | 14 ++++++++ .../connectors/source-exchange-rates/main.py | 9 ++--- .../connectors/source-exchange-rates/setup.py | 19 +++++++++- .../source_exchange_rates/run.py | 14 ++++++++ .../connectors/source-facebook-pages/main.py | 9 ++--- .../connectors/source-facebook-pages/setup.py | 19 +++++++++- .../source_facebook_pages/run.py | 14 ++++++++ .../connectors/source-fastbill/main.py | 9 ++--- .../connectors/source-fastbill/setup.py | 19 +++++++++- .../source-fastbill/source_fastbill/run.py | 14 ++++++++ .../connectors/source-fauna/main.py | 9 ++--- .../connectors/source-fauna/setup.py | 5 +++ .../source-fauna/source_fauna/run.py | 14 ++++++++ .../connectors/source-file/main.py | 9 ++--- .../connectors/source-file/setup.py | 5 +++ .../connectors/source-file/source_file/run.py | 14 ++++++++ .../source-firebase-realtime-database/main.py | 9 ++--- .../setup.py | 5 +++ .../source_firebase_realtime_database/run.py | 14 ++++++++ .../connectors/source-firebolt/main.py | 9 ++--- .../connectors/source-firebolt/setup.py | 5 +++ .../source-firebolt/source_firebolt/run.py | 14 ++++++++ .../connectors/source-flexport/main.py | 9 ++--- .../connectors/source-flexport/setup.py | 19 +++++++++- .../source-flexport/source_flexport/run.py | 14 ++++++++ .../connectors/source-freshcaller/main.py | 9 ++--- .../connectors/source-freshcaller/setup.py | 19 +++++++++- .../source_freshcaller/run.py | 14 ++++++++ .../connectors/source-freshsales/main.py | 9 ++--- .../connectors/source-freshsales/setup.py | 19 +++++++++- .../source_freshsales/run.py | 14 ++++++++ .../connectors/source-freshservice/main.py | 9 ++--- .../connectors/source-freshservice/setup.py | 19 +++++++++- .../source_freshservice/run.py | 14 ++++++++ .../connectors/source-fullstory/main.py | 9 ++--- .../connectors/source-fullstory/setup.py | 19 +++++++++- .../source-fullstory/source_fullstory/run.py | 14 ++++++++ .../connectors/source-gainsight-px/main.py | 9 ++--- .../connectors/source-gainsight-px/setup.py | 19 +++++++++- .../source_gainsight_px/run.py | 14 ++++++++ .../connectors/source-gcs/main.py | 11 ++---- .../connectors/source-gcs/setup.py | 5 +++ .../connectors/source-gcs/source_gcs/run.py | 16 +++++++++ .../connectors/source-genesys/main.py | 9 ++--- .../connectors/source-genesys/setup.py | 19 +++++++++- .../source-genesys/source_genesys/run.py | 14 ++++++++ .../connectors/source-getlago/main.py | 9 ++--- .../connectors/source-getlago/setup.py | 19 +++++++++- .../source-getlago/source_getlago/run.py | 14 ++++++++ .../connectors/source-github/main.py | 12 ++----- .../connectors/source-github/setup.py | 5 +++ .../source-github/source_github/run.py | 17 +++++++++ .../connectors/source-glassfrog/main.py | 9 ++--- .../connectors/source-glassfrog/setup.py | 19 +++++++++- .../source-glassfrog/source_glassfrog/run.py | 14 ++++++++ .../connectors/source-gnews/main.py | 9 ++--- .../connectors/source-gnews/setup.py | 19 +++++++++- .../source-gnews/source_gnews/run.py | 14 ++++++++ .../connectors/source-gocardless/main.py | 9 ++--- .../connectors/source-gocardless/setup.py | 19 +++++++++- .../source_gocardless/run.py | 14 ++++++++ .../connectors/source-gong/main.py | 9 ++--- .../connectors/source-gong/setup.py | 19 +++++++++- .../connectors/source-gong/source_gong/run.py | 14 ++++++++ .../source-google-analytics-v4/main.py | 9 ++--- .../source-google-analytics-v4/setup.py | 5 +++ .../source_google_analytics_v4/run.py | 14 ++++++++ .../source-google-directory/main.py | 9 ++--- .../source-google-directory/setup.py | 5 +++ .../source_google_directory/run.py | 14 ++++++++ .../source-google-pagespeed-insights/main.py | 9 ++--- .../source-google-pagespeed-insights/setup.py | 19 +++++++++- .../source_google_pagespeed_insights/run.py | 14 ++++++++ .../source-google-search-console/main.py | 13 ++----- .../source-google-search-console/setup.py | 5 +++ .../source_google_search_console/run.py | 18 ++++++++++ .../connectors/source-google-sheets/setup.py | 14 +++++++- .../connectors/source-google-webfonts/main.py | 9 ++--- .../source-google-webfonts/setup.py | 19 +++++++++- .../source_google_webfonts/run.py | 14 ++++++++ .../main.py | 9 ++--- .../setup.py | 5 +++ .../run.py | 14 ++++++++ .../connectors/source-greenhouse/main.py | 9 ++--- .../connectors/source-greenhouse/setup.py | 5 +++ .../source_greenhouse/run.py | 14 ++++++++ .../connectors/source-gridly/main.py | 9 ++--- .../connectors/source-gridly/setup.py | 19 +++++++++- .../source-gridly/source_gridly/run.py | 14 ++++++++ .../connectors/source-gutendex/main.py | 9 ++--- .../connectors/source-gutendex/setup.py | 19 +++++++++- .../source-gutendex/source_gutendex/run.py | 14 ++++++++ .../connectors/source-harness/main.py | 9 ++--- .../connectors/source-harness/setup.py | 19 +++++++++- .../source-harness/source_harness/run.py | 14 ++++++++ .../connectors/source-harvest/main.py | 9 ++--- .../connectors/source-harvest/setup.py | 5 +++ .../source-harvest/source_harvest/run.py | 14 ++++++++ .../connectors/source-hellobaton/main.py | 9 ++--- .../connectors/source-hellobaton/setup.py | 19 +++++++++- .../source_hellobaton/run.py | 14 ++++++++ .../connectors/source-hubplanner/main.py | 9 ++--- .../connectors/source-hubplanner/setup.py | 19 +++++++++- .../source_hubplanner/run.py | 14 ++++++++ .../connectors/source-hubspot/main.py | 9 ++--- .../connectors/source-hubspot/setup.py | 5 +++ .../source-hubspot/source_hubspot/run.py | 14 ++++++++ .../connectors/source-insightly/main.py | 9 ++--- .../connectors/source-insightly/setup.py | 19 +++++++++- .../source-insightly/source_insightly/run.py | 14 ++++++++ .../connectors/source-instatus/main.py | 9 ++--- .../connectors/source-instatus/setup.py | 19 +++++++++- .../source-instatus/source_instatus/run.py | 14 ++++++++ .../connectors/source-intercom/main.py | 9 ++--- .../connectors/source-intercom/setup.py | 19 +++++++++- .../source-intercom/source_intercom/run.py | 14 ++++++++ .../connectors/source-intruder/main.py | 9 ++--- .../connectors/source-intruder/setup.py | 19 +++++++++- .../source-intruder/source_intruder/run.py | 14 ++++++++ .../connectors/source-ip2whois/main.py | 9 ++--- .../connectors/source-ip2whois/setup.py | 19 +++++++++- .../source-ip2whois/source_ip2whois/run.py | 14 ++++++++ .../connectors/source-jira/main.py | 9 ++--- .../connectors/source-jira/setup.py | 5 +++ .../connectors/source-jira/source_jira/run.py | 14 ++++++++ .../connectors/source-k6-cloud/main.py | 9 ++--- .../connectors/source-k6-cloud/setup.py | 19 +++++++++- .../source-k6-cloud/source_k6_cloud/run.py | 14 ++++++++ .../connectors/source-klarna/main.py | 9 ++--- .../connectors/source-klarna/setup.py | 19 +++++++++- .../source-klarna/source_klarna/run.py | 14 ++++++++ .../connectors/source-klaus-api/main.py | 9 ++--- .../connectors/source-klaus-api/setup.py | 19 +++++++++- .../source-klaus-api/source_klaus_api/run.py | 14 ++++++++ .../connectors/source-klaviyo/main.py | 9 ++--- .../connectors/source-klaviyo/setup.py | 5 +++ .../source-klaviyo/source_klaviyo/run.py | 14 ++++++++ .../connectors/source-kustomer-singer/main.py | 8 ++--- .../source-kustomer-singer/setup.py | 5 +++ .../source_kustomer_singer/run.py | 13 +++++++ .../connectors/source-kyriba/main.py | 9 ++--- .../connectors/source-kyriba/setup.py | 5 +++ .../source-kyriba/source_kyriba/run.py | 14 ++++++++ .../connectors/source-kyve/main.py | 9 ++--- .../connectors/source-kyve/setup.py | 5 +++ .../connectors/source-kyve/source_kyve/run.py | 14 ++++++++ .../connectors/source-launchdarkly/main.py | 9 ++--- .../connectors/source-launchdarkly/setup.py | 19 +++++++++- .../source_launchdarkly/run.py | 14 ++++++++ .../connectors/source-lemlist/main.py | 9 ++--- .../connectors/source-lemlist/setup.py | 19 +++++++++- .../source-lemlist/source_lemlist/run.py | 14 ++++++++ .../connectors/source-lever-hiring/main.py | 9 ++--- .../connectors/source-lever-hiring/setup.py | 5 +++ .../source_lever_hiring/run.py | 14 ++++++++ .../connectors/source-linkedin-pages/main.py | 9 ++--- .../connectors/source-linkedin-pages/setup.py | 5 +++ .../source_linkedin_pages/run.py | 14 ++++++++ .../connectors/source-linnworks/main.py | 9 ++--- .../connectors/source-linnworks/setup.py | 5 +++ .../source-linnworks/source_linnworks/run.py | 14 ++++++++ .../connectors/source-lokalise/main.py | 9 ++--- .../connectors/source-lokalise/setup.py | 19 +++++++++- .../source-lokalise/source_lokalise/run.py | 14 ++++++++ .../connectors/source-looker/main.py | 9 ++--- .../connectors/source-looker/setup.py | 5 +++ .../source-looker/source_looker/run.py | 14 ++++++++ .../connectors/source-mailerlite/main.py | 9 ++--- .../connectors/source-mailerlite/setup.py | 19 +++++++++- .../source_mailerlite/run.py | 14 ++++++++ .../connectors/source-mailersend/main.py | 9 ++--- .../connectors/source-mailersend/setup.py | 19 +++++++++- .../source_mailersend/run.py | 14 ++++++++ .../connectors/source-mailgun/main.py | 9 ++--- .../connectors/source-mailgun/setup.py | 19 +++++++++- .../source-mailgun/source_mailgun/run.py | 14 ++++++++ .../connectors/source-mailjet-mail/main.py | 9 ++--- .../connectors/source-mailjet-mail/setup.py | 19 +++++++++- .../source_mailjet_mail/run.py | 14 ++++++++ .../connectors/source-mailjet-sms/main.py | 9 ++--- .../connectors/source-mailjet-sms/setup.py | 19 +++++++++- .../source_mailjet_sms/run.py | 14 ++++++++ .../connectors/source-merge/main.py | 9 ++--- .../connectors/source-merge/setup.py | 19 +++++++++- .../source-merge/source_merge/run.py | 14 ++++++++ .../connectors/source-metabase/main.py | 9 ++--- .../connectors/source-metabase/setup.py | 19 +++++++++- .../source-metabase/source_metabase/run.py | 14 ++++++++ .../source-microsoft-dataverse/main.py | 9 ++--- .../source-microsoft-dataverse/setup.py | 19 +++++++++- .../source_microsoft_dataverse/run.py | 14 ++++++++ .../source-microsoft-onedrive/main.py | 12 ++----- .../source-microsoft-onedrive/setup.py | 19 +++++++++- .../source_microsoft_onedrive/run.py | 17 +++++++++ .../connectors/source-microsoft-teams/main.py | 9 ++--- .../source-microsoft-teams/setup.py | 5 +++ .../source_microsoft_teams/run.py | 14 ++++++++ .../connectors/source-monday/main.py | 9 ++--- .../connectors/source-monday/setup.py | 5 +++ .../source-monday/source_monday/run.py | 14 ++++++++ .../connectors/source-my-hours/main.py | 9 ++--- .../connectors/source-my-hours/setup.py | 5 +++ .../source-my-hours/source_my_hours/run.py | 14 ++++++++ .../connectors/source-n8n/main.py | 9 ++--- .../connectors/source-n8n/setup.py | 19 +++++++++- .../connectors/source-n8n/source_n8n/run.py | 14 ++++++++ .../connectors/source-nasa/main.py | 9 ++--- .../connectors/source-nasa/setup.py | 19 +++++++++- .../connectors/source-nasa/source_nasa/run.py | 14 ++++++++ .../connectors/source-netsuite/main.py | 9 ++--- .../connectors/source-netsuite/setup.py | 19 +++++++++- .../source-netsuite/source_netsuite/run.py | 14 ++++++++ .../connectors/source-news-api/main.py | 9 ++--- .../connectors/source-news-api/setup.py | 19 +++++++++- .../source-news-api/source_news_api/run.py | 14 ++++++++ .../connectors/source-newsdata/main.py | 9 ++--- .../connectors/source-newsdata/setup.py | 19 +++++++++- .../source-newsdata/source_newsdata/run.py | 14 ++++++++ .../connectors/source-notion/main.py | 9 ++--- .../connectors/source-notion/setup.py | 5 +++ .../source-notion/source_notion/run.py | 14 ++++++++ .../connectors/source-nytimes/main.py | 9 ++--- .../connectors/source-nytimes/setup.py | 19 +++++++++- .../source-nytimes/source_nytimes/run.py | 14 ++++++++ .../connectors/source-okta/main.py | 9 ++--- .../connectors/source-okta/setup.py | 5 +++ .../connectors/source-okta/source_okta/run.py | 14 ++++++++ .../connectors/source-omnisend/main.py | 9 ++--- .../connectors/source-omnisend/setup.py | 19 +++++++++- .../source-omnisend/source_omnisend/run.py | 14 ++++++++ .../connectors/source-onesignal/main.py | 9 ++--- .../connectors/source-onesignal/setup.py | 19 +++++++++- .../source-onesignal/source_onesignal/run.py | 14 ++++++++ .../source-open-exchange-rates/main.py | 9 ++--- .../source-open-exchange-rates/setup.py | 19 +++++++++- .../source_open_exchange_rates/run.py | 14 ++++++++ .../connectors/source-openweather/main.py | 9 ++--- .../connectors/source-openweather/setup.py | 19 +++++++++- .../source_openweather/run.py | 14 ++++++++ .../connectors/source-opsgenie/main.py | 9 ++--- .../connectors/source-opsgenie/setup.py | 19 +++++++++- .../source-opsgenie/source_opsgenie/run.py | 14 ++++++++ .../connectors/source-orb/main.py | 9 ++--- .../connectors/source-orb/setup.py | 5 +++ .../connectors/source-orb/source_orb/run.py | 14 ++++++++ .../connectors/source-orbit/main.py | 9 ++--- .../connectors/source-orbit/setup.py | 19 +++++++++- .../source-orbit/source_orbit/run.py | 14 ++++++++ .../connectors/source-oura/main.py | 9 ++--- .../connectors/source-oura/setup.py | 19 +++++++++- .../connectors/source-oura/source_oura/run.py | 14 ++++++++ .../source-outbrain-amplify/main.py | 9 ++--- .../source-outbrain-amplify/setup.py | 19 +++++++++- .../source_outbrain_amplify/run.py | 14 ++++++++ .../connectors/source-outreach/main.py | 9 ++--- .../connectors/source-outreach/setup.py | 5 +++ .../source-outreach/source_outreach/run.py | 14 ++++++++ .../connectors/source-pagerduty/main.py | 9 ++--- .../connectors/source-pagerduty/setup.py | 19 +++++++++- .../source-pagerduty/source_pagerduty/run.py | 14 ++++++++ .../connectors/source-pardot/main.py | 9 ++--- .../connectors/source-pardot/setup.py | 5 +++ .../source-pardot/source_pardot/run.py | 14 ++++++++ .../connectors/source-partnerstack/main.py | 9 ++--- .../connectors/source-partnerstack/setup.py | 19 +++++++++- .../source_partnerstack/run.py | 14 ++++++++ .../source-paypal-transaction/setup.py | 14 +++++++- .../connectors/source-paystack/main.py | 9 ++--- .../connectors/source-paystack/setup.py | 5 +++ .../source-paystack/source_paystack/run.py | 14 ++++++++ .../connectors/source-pendo/main.py | 9 ++--- .../connectors/source-pendo/setup.py | 19 +++++++++- .../source-pendo/source_pendo/run.py | 14 ++++++++ .../connectors/source-persistiq/main.py | 9 ++--- .../connectors/source-persistiq/setup.py | 19 +++++++++- .../source-persistiq/source_persistiq/run.py | 14 ++++++++ .../connectors/source-pexels-api/main.py | 9 ++--- .../connectors/source-pexels-api/setup.py | 19 +++++++++- .../source_pexels_api/run.py | 14 ++++++++ .../connectors/source-pinterest/main.py | 9 ++--- .../connectors/source-pinterest/setup.py | 5 +++ .../source-pinterest/source_pinterest/run.py | 14 ++++++++ .../connectors/source-pipedrive/setup.py | 14 +++++++- .../connectors/source-pivotal-tracker/main.py | 9 ++--- .../source-pivotal-tracker/setup.py | 5 +++ .../source_pivotal_tracker/run.py | 14 ++++++++ .../connectors/source-plaid/main.py | 9 ++--- .../connectors/source-plaid/setup.py | 19 +++++++++- .../source-plaid/source_plaid/run.py | 14 ++++++++ .../connectors/source-plausible/main.py | 9 ++--- .../connectors/source-plausible/setup.py | 19 +++++++++- .../source-plausible/source_plausible/run.py | 14 ++++++++ .../connectors/source-pocket/main.py | 9 ++--- .../connectors/source-pocket/setup.py | 19 +++++++++- .../source-pocket/source_pocket/run.py | 14 ++++++++ .../connectors/source-pokeapi/main.py | 8 ++--- .../connectors/source-pokeapi/setup.py | 19 +++++++++- .../source-pokeapi/source_pokeapi/run.py | 13 +++++++ .../source-polygon-stock-api/main.py | 9 ++--- .../source-polygon-stock-api/setup.py | 19 +++++++++- .../source_polygon_stock_api/run.py | 14 ++++++++ .../connectors/source-posthog/main.py | 9 ++--- .../connectors/source-posthog/setup.py | 5 +++ .../source-posthog/source_posthog/run.py | 14 ++++++++ .../connectors/source-postmarkapp/main.py | 9 ++--- .../connectors/source-postmarkapp/setup.py | 19 +++++++++- .../source_postmarkapp/run.py | 14 ++++++++ .../connectors/source-prestashop/main.py | 9 ++--- .../connectors/source-prestashop/setup.py | 19 +++++++++- .../source_prestashop/run.py | 14 ++++++++ .../connectors/source-primetric/main.py | 9 ++--- .../connectors/source-primetric/setup.py | 19 +++++++++- .../source-primetric/source_primetric/run.py | 14 ++++++++ .../connectors/source-public-apis/main.py | 9 ++--- .../connectors/source-public-apis/setup.py | 19 +++++++++- .../source_public_apis/run.py | 14 ++++++++ .../connectors/source-punk-api/main.py | 9 ++--- .../connectors/source-punk-api/setup.py | 19 +++++++++- .../source-punk-api/source_punk_api/run.py | 14 ++++++++ .../connectors/source-pypi/main.py | 9 ++--- .../connectors/source-pypi/setup.py | 19 +++++++++- .../connectors/source-pypi/source_pypi/run.py | 14 ++++++++ .../source-python-http-tutorial/main.py | 9 ++--- .../source-python-http-tutorial/setup.py | 5 +++ .../source_python_http_tutorial/run.py | 14 ++++++++ .../connectors/source-qonto/main.py | 9 ++--- .../connectors/source-qonto/setup.py | 19 +++++++++- .../source-qonto/source_qonto/run.py | 14 ++++++++ .../connectors/source-qualaroo/main.py | 9 ++--- .../connectors/source-qualaroo/setup.py | 19 +++++++++- .../source-qualaroo/source_qualaroo/run.py | 14 ++++++++ .../connectors/source-quickbooks/main.py | 9 ++--- .../connectors/source-quickbooks/setup.py | 19 +++++++++- .../source_quickbooks/run.py | 14 ++++++++ .../connectors/source-railz/main.py | 9 ++--- .../connectors/source-railz/setup.py | 19 +++++++++- .../source-railz/source_railz/run.py | 14 ++++++++ .../source-rd-station-marketing/main.py | 9 ++--- .../source-rd-station-marketing/setup.py | 19 +++++++++- .../source_rd_station_marketing/run.py | 14 ++++++++ .../connectors/source-recharge/main.py | 9 ++--- .../connectors/source-recharge/setup.py | 5 +++ .../source-recharge/source_recharge/run.py | 14 ++++++++ .../connectors/source-recreation/main.py | 9 ++--- .../connectors/source-recreation/setup.py | 19 +++++++++- .../source_recreation/run.py | 14 ++++++++ .../connectors/source-recruitee/main.py | 9 ++--- .../connectors/source-recruitee/setup.py | 19 +++++++++- .../source-recruitee/source_recruitee/run.py | 14 ++++++++ .../connectors/source-recurly/main.py | 9 ++--- .../connectors/source-recurly/setup.py | 5 +++ .../source-recurly/source_recurly/run.py | 14 ++++++++ .../connectors/source-reply-io/main.py | 9 ++--- .../connectors/source-reply-io/setup.py | 19 +++++++++- .../source-reply-io/source_reply_io/run.py | 14 ++++++++ .../connectors/source-retently/main.py | 9 ++--- .../connectors/source-retently/setup.py | 19 +++++++++- .../source-retently/source_retently/run.py | 14 ++++++++ .../connectors/source-ringcentral/main.py | 9 ++--- .../connectors/source-ringcentral/setup.py | 19 +++++++++- .../source_ringcentral/run.py | 14 ++++++++ .../connectors/source-rki-covid/main.py | 9 ++--- .../connectors/source-rki-covid/setup.py | 5 +++ .../source-rki-covid/source_rki_covid/run.py | 14 ++++++++ .../connectors/source-rocket-chat/main.py | 9 ++--- .../connectors/source-rocket-chat/setup.py | 19 +++++++++- .../source_rocket_chat/run.py | 14 ++++++++ .../connectors/source-rss/main.py | 9 ++--- .../connectors/source-rss/setup.py | 19 +++++++++- .../connectors/source-rss/source_rss/run.py | 14 ++++++++ .../connectors/source-salesforce/setup.py | 14 +++++++- .../connectors/source-salesloft/main.py | 9 ++--- .../connectors/source-salesloft/setup.py | 5 +++ .../source-salesloft/source_salesloft/run.py | 14 ++++++++ .../connectors/source-sap-fieldglass/main.py | 9 ++--- .../connectors/source-sap-fieldglass/setup.py | 19 +++++++++- .../source_sap_fieldglass/run.py | 14 ++++++++ .../source-scaffold-source-http/main.py | 9 ++--- .../source-scaffold-source-http/setup.py | 19 +++++++++- .../source_scaffold_source_http/run.py | 14 ++++++++ .../source-scaffold-source-python/main.py | 9 ++--- .../source-scaffold-source-python/setup.py | 5 +++ .../source_scaffold_source_python/run.py | 14 ++++++++ .../connectors/source-search-metrics/main.py | 9 ++--- .../connectors/source-search-metrics/setup.py | 5 +++ .../source_search_metrics/run.py | 14 ++++++++ .../connectors/source-secoda/main.py | 9 ++--- .../connectors/source-secoda/setup.py | 19 +++++++++- .../source-secoda/source_secoda/run.py | 14 ++++++++ .../connectors/source-sendgrid/main.py | 9 ++--- .../connectors/source-sendgrid/setup.py | 5 +++ .../source-sendgrid/source_sendgrid/run.py | 14 ++++++++ .../connectors/source-sendinblue/main.py | 9 ++--- .../connectors/source-sendinblue/setup.py | 19 +++++++++- .../source_sendinblue/run.py | 14 ++++++++ .../connectors/source-senseforce/main.py | 9 ++--- .../connectors/source-senseforce/setup.py | 19 +++++++++- .../source_senseforce/run.py | 14 ++++++++ .../connectors/source-sentry/main.py | 9 ++--- .../connectors/source-sentry/setup.py | 5 +++ .../source-sentry/source_sentry/run.py | 14 ++++++++ .../connectors/source-serpstat/main.py | 9 ++--- .../connectors/source-serpstat/setup.py | 19 +++++++++- .../source-serpstat/source_serpstat/run.py | 14 ++++++++ .../connectors/source-sftp-bulk/main.py | 9 ++--- .../connectors/source-sftp-bulk/setup.py | 5 +++ .../source-sftp-bulk/source_sftp_bulk/run.py | 14 ++++++++ .../connectors/source-shortio/main.py | 9 ++--- .../connectors/source-shortio/setup.py | 19 +++++++++- .../source-shortio/source_shortio/run.py | 14 ++++++++ .../connectors/source-smaily/main.py | 9 ++--- .../connectors/source-smaily/setup.py | 19 +++++++++- .../source-smaily/source_smaily/run.py | 14 ++++++++ .../connectors/source-smartengage/main.py | 9 ++--- .../connectors/source-smartengage/setup.py | 19 +++++++++- .../source_smartengage/run.py | 14 ++++++++ .../source-snapchat-marketing/main.py | 9 ++--- .../source-snapchat-marketing/setup.py | 5 +++ .../source_snapchat_marketing/run.py | 14 ++++++++ .../connectors/source-sonar-cloud/main.py | 9 ++--- .../connectors/source-sonar-cloud/setup.py | 19 +++++++++- .../source_sonar_cloud/run.py | 14 ++++++++ .../connectors/source-spacex-api/main.py | 9 ++--- .../connectors/source-spacex-api/setup.py | 19 +++++++++- .../source_spacex_api/run.py | 14 ++++++++ .../connectors/source-square/main.py | 9 ++--- .../connectors/source-square/setup.py | 19 +++++++++- .../source-square/source_square/run.py | 14 ++++++++ .../connectors/source-statuspage/main.py | 9 ++--- .../connectors/source-statuspage/setup.py | 19 +++++++++- .../source_statuspage/run.py | 14 ++++++++ .../connectors/source-strava/main.py | 9 ++--- .../connectors/source-strava/setup.py | 19 +++++++++- .../source-strava/source_strava/run.py | 14 ++++++++ .../connectors/source-stripe/setup.py | 14 +++++++- .../connectors/source-survey-sparrow/main.py | 9 ++--- .../connectors/source-survey-sparrow/setup.py | 19 +++++++++- .../source_survey_sparrow/run.py | 14 ++++++++ .../connectors/source-surveycto/main.py | 9 ++--- .../connectors/source-surveycto/setup.py | 19 +++++++++- .../source-surveycto/source_surveycto/run.py | 14 ++++++++ .../connectors/source-surveymonkey/main.py | 9 ++--- .../connectors/source-surveymonkey/setup.py | 5 +++ .../source_surveymonkey/run.py | 14 ++++++++ .../source-talkdesk-explore/main.py | 9 ++--- .../source-talkdesk-explore/setup.py | 5 +++ .../source_talkdesk_explore/run.py | 14 ++++++++ .../connectors/source-tempo/main.py | 9 ++--- .../connectors/source-tempo/setup.py | 5 +++ .../source-tempo/source_tempo/run.py | 14 ++++++++ .../source-the-guardian-api/main.py | 9 ++--- .../source-the-guardian-api/setup.py | 19 +++++++++- .../source_the_guardian_api/run.py | 14 ++++++++ .../source-tiktok-marketing/main.py | 9 ++--- .../source-tiktok-marketing/setup.py | 5 +++ .../source_tiktok_marketing/run.py | 14 ++++++++ .../connectors/source-timely/main.py | 9 ++--- .../connectors/source-timely/setup.py | 19 +++++++++- .../source-timely/source_timely/run.py | 14 ++++++++ .../connectors/source-tmdb/main.py | 9 ++--- .../connectors/source-tmdb/setup.py | 19 +++++++++- .../connectors/source-tmdb/source_tmdb/run.py | 14 ++++++++ .../connectors/source-todoist/main.py | 9 ++--- .../connectors/source-todoist/setup.py | 19 +++++++++- .../source-todoist/source_todoist/run.py | 14 ++++++++ .../connectors/source-toggl/main.py | 9 ++--- .../connectors/source-toggl/setup.py | 19 +++++++++- .../source-toggl/source_toggl/run.py | 14 ++++++++ .../connectors/source-tplcentral/main.py | 9 ++--- .../connectors/source-tplcentral/setup.py | 5 +++ .../source_tplcentral/run.py | 14 ++++++++ .../connectors/source-trello/main.py | 9 ++--- .../connectors/source-trello/setup.py | 19 +++++++++- .../source-trello/source_trello/run.py | 14 ++++++++ .../connectors/source-trustpilot/main.py | 9 ++--- .../connectors/source-trustpilot/setup.py | 19 +++++++++- .../source_trustpilot/run.py | 14 ++++++++ .../connectors/source-tvmaze-schedule/main.py | 9 ++--- .../source-tvmaze-schedule/setup.py | 19 +++++++++- .../source_tvmaze_schedule/run.py | 14 ++++++++ .../source-twilio-taskrouter/main.py | 9 ++--- .../source-twilio-taskrouter/setup.py | 19 +++++++++- .../source_twilio_taskrouter/run.py | 14 ++++++++ .../connectors/source-twilio/main.py | 9 ++--- .../connectors/source-twilio/setup.py | 5 +++ .../source-twilio/source_twilio/run.py | 14 ++++++++ .../connectors/source-twitter/main.py | 9 ++--- .../connectors/source-twitter/setup.py | 19 +++++++++- .../source-twitter/source_twitter/run.py | 14 ++++++++ .../connectors/source-tyntec-sms/main.py | 9 ++--- .../connectors/source-tyntec-sms/setup.py | 19 +++++++++- .../source_tyntec_sms/run.py | 14 ++++++++ .../connectors/source-typeform/setup.py | 14 +++++++- .../connectors/source-unleash/main.py | 9 ++--- .../connectors/source-unleash/setup.py | 19 +++++++++- .../source-unleash/source_unleash/run.py | 14 ++++++++ .../connectors/source-us-census/main.py | 9 ++--- .../connectors/source-us-census/setup.py | 5 +++ .../source-us-census/source_us_census/run.py | 14 ++++++++ .../connectors/source-vantage/main.py | 9 ++--- .../connectors/source-vantage/setup.py | 19 +++++++++- .../source-vantage/source_vantage/run.py | 14 ++++++++ .../connectors/source-visma-economic/main.py | 9 ++--- .../connectors/source-visma-economic/setup.py | 19 +++++++++- .../source_visma_economic/run.py | 14 ++++++++ .../connectors/source-vitally/main.py | 9 ++--- .../connectors/source-vitally/setup.py | 19 +++++++++- .../source-vitally/source_vitally/run.py | 14 ++++++++ .../connectors/source-waiteraid/main.py | 9 ++--- .../connectors/source-waiteraid/setup.py | 19 +++++++++- .../source-waiteraid/source_waiteraid/run.py | 14 ++++++++ .../connectors/source-weatherstack/main.py | 9 ++--- .../connectors/source-weatherstack/setup.py | 19 +++++++++- .../source_weatherstack/run.py | 14 ++++++++ .../connectors/source-webflow/main.py | 9 ++--- .../connectors/source-webflow/setup.py | 19 +++++++++- .../source-webflow/source_webflow/run.py | 14 ++++++++ .../connectors/source-whisky-hunter/main.py | 9 ++--- .../connectors/source-whisky-hunter/setup.py | 19 +++++++++- .../source_whisky_hunter/run.py | 14 ++++++++ .../source-wikipedia-pageviews/main.py | 9 ++--- .../source-wikipedia-pageviews/setup.py | 19 +++++++++- .../source_wikipedia_pageviews/run.py | 14 ++++++++ .../connectors/source-woocommerce/main.py | 9 ++--- .../connectors/source-woocommerce/setup.py | 19 +++++++++- .../source_woocommerce/run.py | 14 ++++++++ .../connectors/source-workable/main.py | 9 ++--- .../connectors/source-workable/setup.py | 19 +++++++++- .../source-workable/source_workable/run.py | 14 ++++++++ .../connectors/source-workramp/main.py | 9 ++--- .../connectors/source-workramp/setup.py | 19 +++++++++- .../source-workramp/source_workramp/run.py | 14 ++++++++ .../connectors/source-wrike/main.py | 9 ++--- .../connectors/source-wrike/setup.py | 19 +++++++++- .../source-wrike/source_wrike/run.py | 14 ++++++++ .../connectors/source-xero/setup.py | 14 +++++++- .../connectors/source-xkcd/main.py | 9 ++--- .../connectors/source-xkcd/setup.py | 19 +++++++++- .../connectors/source-xkcd/source_xkcd/run.py | 14 ++++++++ .../source-yahoo-finance-price/main.py | 9 ++--- .../source-yahoo-finance-price/setup.py | 19 +++++++++- .../source_yahoo_finance_price/run.py | 14 ++++++++ .../connectors/source-yandex-metrica/main.py | 9 ++--- .../connectors/source-yandex-metrica/setup.py | 19 +++++++++- .../source_yandex_metrica/run.py | 14 ++++++++ .../connectors/source-yotpo/main.py | 9 ++--- .../connectors/source-yotpo/setup.py | 19 +++++++++- .../source-yotpo/source_yotpo/run.py | 14 ++++++++ .../connectors/source-younium/main.py | 9 ++--- .../connectors/source-younium/setup.py | 19 +++++++++- .../source-younium/source_younium/run.py | 14 ++++++++ .../source-youtube-analytics/main.py | 9 ++--- .../source-youtube-analytics/setup.py | 5 +++ .../source_youtube_analytics/run.py | 14 ++++++++ .../source-zapier-supported-storage/main.py | 9 ++--- .../source-zapier-supported-storage/setup.py | 19 +++++++++- .../source_zapier_supported_storage/run.py | 14 ++++++++ .../connectors/source-zendesk-sell/main.py | 9 ++--- .../connectors/source-zendesk-sell/setup.py | 19 +++++++++- .../source_zendesk_sell/run.py | 14 ++++++++ .../source-zendesk-sunshine/main.py | 9 ++--- .../source-zendesk-sunshine/setup.py | 19 +++++++++- .../source_zendesk_sunshine/run.py | 14 ++++++++ .../connectors/source-zenefits/main.py | 9 ++--- .../connectors/source-zenefits/setup.py | 19 +++++++++- .../source-zenefits/source_zenefits/run.py | 14 ++++++++ .../connectors/source-zenloop/main.py | 9 ++--- .../connectors/source-zenloop/setup.py | 5 +++ .../source-zenloop/source_zenloop/run.py | 14 ++++++++ .../connectors/source-zoho-crm/main.py | 9 ++--- .../connectors/source-zoho-crm/setup.py | 5 +++ .../source-zoho-crm/source_zoho_crm/run.py | 14 ++++++++ .../connectors/source-zoom/main.py | 9 ++--- .../connectors/source-zoom/setup.py | 19 +++++++++- .../connectors/source-zoom/source_zoom/run.py | 14 ++++++++ .../connectors/source-zuora/main.py | 9 ++--- .../connectors/source-zuora/setup.py | 5 +++ .../source-zuora/source_zuora/run.py | 14 ++++++++ 755 files changed, 7678 insertions(+), 1984 deletions(-) create mode 100644 airbyte-integrations/connectors/source-activecampaign/source_activecampaign/run.py create mode 100644 airbyte-integrations/connectors/source-adjust/source_adjust/run.py create mode 100644 airbyte-integrations/connectors/source-aha/source_aha/run.py create mode 100644 airbyte-integrations/connectors/source-aircall/source_aircall/run.py create mode 100644 airbyte-integrations/connectors/source-airtable/source_airtable/run.py create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/run.py create mode 100644 airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py create mode 100644 airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/run.py create mode 100644 airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/run.py create mode 100644 airbyte-integrations/connectors/source-amplitude/source_amplitude/run.py create mode 100644 airbyte-integrations/connectors/source-appfollow/source_appfollow/run.py create mode 100644 airbyte-integrations/connectors/source-apple-search-ads/source_apple_search_ads/run.py create mode 100644 airbyte-integrations/connectors/source-appsflyer/source_appsflyer/run.py create mode 100644 airbyte-integrations/connectors/source-appstore-singer/source_appstore_singer/run.py create mode 100644 airbyte-integrations/connectors/source-asana/source_asana/run.py create mode 100644 airbyte-integrations/connectors/source-ashby/source_ashby/run.py create mode 100644 airbyte-integrations/connectors/source-auth0/source_auth0/run.py create mode 100644 airbyte-integrations/connectors/source-aws-cloudtrail/source_aws_cloudtrail/run.py create mode 100644 airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/run.py create mode 100644 airbyte-integrations/connectors/source-azure-table/source_azure_table/run.py create mode 100644 airbyte-integrations/connectors/source-babelforce/source_babelforce/run.py create mode 100644 airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/run.py create mode 100644 airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/run.py create mode 100644 airbyte-integrations/connectors/source-bing-ads/source_bing_ads/run.py create mode 100644 airbyte-integrations/connectors/source-braintree/source_braintree/run.py create mode 100644 airbyte-integrations/connectors/source-braze/source_braze/run.py create mode 100644 airbyte-integrations/connectors/source-breezometer/source_breezometer/run.py create mode 100644 airbyte-integrations/connectors/source-callrail/source_callrail/run.py create mode 100644 airbyte-integrations/connectors/source-captain-data/source_captain_data/run.py create mode 100644 airbyte-integrations/connectors/source-cart/source_cart/run.py create mode 100644 airbyte-integrations/connectors/source-chargebee/source_chargebee/run.py create mode 100644 airbyte-integrations/connectors/source-chargify/source_chargify/run.py create mode 100644 airbyte-integrations/connectors/source-chartmogul/source_chartmogul/run.py create mode 100644 airbyte-integrations/connectors/source-clickup-api/source_clickup_api/run.py create mode 100644 airbyte-integrations/connectors/source-clockify/source_clockify/run.py create mode 100644 airbyte-integrations/connectors/source-close-com/source_close_com/run.py create mode 100644 airbyte-integrations/connectors/source-coda/source_coda/run.py create mode 100644 airbyte-integrations/connectors/source-coin-api/source_coin_api/run.py create mode 100644 airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/run.py create mode 100644 airbyte-integrations/connectors/source-coinmarketcap/source_coinmarketcap/run.py create mode 100644 airbyte-integrations/connectors/source-commcare/source_commcare/run.py create mode 100644 airbyte-integrations/connectors/source-commercetools/source_commercetools/run.py create mode 100644 airbyte-integrations/connectors/source-configcat/source_configcat/run.py create mode 100644 airbyte-integrations/connectors/source-confluence/source_confluence/run.py create mode 100644 airbyte-integrations/connectors/source-convertkit/source_convertkit/run.py create mode 100644 airbyte-integrations/connectors/source-convex/source_convex/run.py create mode 100644 airbyte-integrations/connectors/source-copper/source_copper/run.py create mode 100644 airbyte-integrations/connectors/source-courier/source_courier/run.py create mode 100644 airbyte-integrations/connectors/source-customer-io/source_customer_io/run.py create mode 100644 airbyte-integrations/connectors/source-datadog/source_datadog/run.py create mode 100644 airbyte-integrations/connectors/source-datascope/source_datascope/run.py create mode 100644 airbyte-integrations/connectors/source-delighted/source_delighted/run.py create mode 100644 airbyte-integrations/connectors/source-dixa/source_dixa/run.py create mode 100644 airbyte-integrations/connectors/source-dockerhub/source_dockerhub/run.py create mode 100644 airbyte-integrations/connectors/source-dremio/source_dremio/run.py create mode 100644 airbyte-integrations/connectors/source-drift/source_drift/run.py create mode 100644 airbyte-integrations/connectors/source-dv-360/source_dv_360/run.py create mode 100644 airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/run.py create mode 100644 airbyte-integrations/connectors/source-everhour/source_everhour/run.py create mode 100644 airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/run.py create mode 100644 airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/run.py create mode 100644 airbyte-integrations/connectors/source-fastbill/source_fastbill/run.py create mode 100644 airbyte-integrations/connectors/source-fauna/source_fauna/run.py create mode 100644 airbyte-integrations/connectors/source-file/source_file/run.py create mode 100644 airbyte-integrations/connectors/source-firebase-realtime-database/source_firebase_realtime_database/run.py create mode 100644 airbyte-integrations/connectors/source-firebolt/source_firebolt/run.py create mode 100644 airbyte-integrations/connectors/source-flexport/source_flexport/run.py create mode 100644 airbyte-integrations/connectors/source-freshcaller/source_freshcaller/run.py create mode 100644 airbyte-integrations/connectors/source-freshsales/source_freshsales/run.py create mode 100644 airbyte-integrations/connectors/source-freshservice/source_freshservice/run.py create mode 100644 airbyte-integrations/connectors/source-fullstory/source_fullstory/run.py create mode 100644 airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/run.py create mode 100644 airbyte-integrations/connectors/source-gcs/source_gcs/run.py create mode 100644 airbyte-integrations/connectors/source-genesys/source_genesys/run.py create mode 100644 airbyte-integrations/connectors/source-getlago/source_getlago/run.py create mode 100644 airbyte-integrations/connectors/source-github/source_github/run.py create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/run.py create mode 100644 airbyte-integrations/connectors/source-gnews/source_gnews/run.py create mode 100644 airbyte-integrations/connectors/source-gocardless/source_gocardless/run.py create mode 100644 airbyte-integrations/connectors/source-gong/source_gong/run.py create mode 100644 airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/run.py create mode 100644 airbyte-integrations/connectors/source-google-directory/source_google_directory/run.py create mode 100644 airbyte-integrations/connectors/source-google-pagespeed-insights/source_google_pagespeed_insights/run.py create mode 100755 airbyte-integrations/connectors/source-google-search-console/source_google_search_console/run.py create mode 100644 airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/run.py create mode 100644 airbyte-integrations/connectors/source-google-workspace-admin-reports/source_google_workspace_admin_reports/run.py create mode 100644 airbyte-integrations/connectors/source-greenhouse/source_greenhouse/run.py create mode 100644 airbyte-integrations/connectors/source-gridly/source_gridly/run.py create mode 100644 airbyte-integrations/connectors/source-gutendex/source_gutendex/run.py create mode 100644 airbyte-integrations/connectors/source-harness/source_harness/run.py create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/run.py create mode 100644 airbyte-integrations/connectors/source-hellobaton/source_hellobaton/run.py create mode 100644 airbyte-integrations/connectors/source-hubplanner/source_hubplanner/run.py create mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/run.py create mode 100644 airbyte-integrations/connectors/source-insightly/source_insightly/run.py create mode 100644 airbyte-integrations/connectors/source-instatus/source_instatus/run.py create mode 100644 airbyte-integrations/connectors/source-intercom/source_intercom/run.py create mode 100644 airbyte-integrations/connectors/source-intruder/source_intruder/run.py create mode 100644 airbyte-integrations/connectors/source-ip2whois/source_ip2whois/run.py create mode 100644 airbyte-integrations/connectors/source-jira/source_jira/run.py create mode 100644 airbyte-integrations/connectors/source-k6-cloud/source_k6_cloud/run.py create mode 100644 airbyte-integrations/connectors/source-klarna/source_klarna/run.py create mode 100644 airbyte-integrations/connectors/source-klaus-api/source_klaus_api/run.py create mode 100644 airbyte-integrations/connectors/source-klaviyo/source_klaviyo/run.py create mode 100644 airbyte-integrations/connectors/source-kustomer-singer/source_kustomer_singer/run.py create mode 100644 airbyte-integrations/connectors/source-kyriba/source_kyriba/run.py create mode 100644 airbyte-integrations/connectors/source-kyve/source_kyve/run.py create mode 100644 airbyte-integrations/connectors/source-launchdarkly/source_launchdarkly/run.py create mode 100644 airbyte-integrations/connectors/source-lemlist/source_lemlist/run.py create mode 100644 airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/run.py create mode 100644 airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/run.py create mode 100644 airbyte-integrations/connectors/source-linnworks/source_linnworks/run.py create mode 100644 airbyte-integrations/connectors/source-lokalise/source_lokalise/run.py create mode 100644 airbyte-integrations/connectors/source-looker/source_looker/run.py create mode 100644 airbyte-integrations/connectors/source-mailerlite/source_mailerlite/run.py create mode 100644 airbyte-integrations/connectors/source-mailersend/source_mailersend/run.py create mode 100644 airbyte-integrations/connectors/source-mailgun/source_mailgun/run.py create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/run.py create mode 100644 airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/run.py create mode 100644 airbyte-integrations/connectors/source-merge/source_merge/run.py create mode 100644 airbyte-integrations/connectors/source-metabase/source_metabase/run.py create mode 100644 airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/run.py create mode 100644 airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/run.py create mode 100644 airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/run.py create mode 100644 airbyte-integrations/connectors/source-monday/source_monday/run.py create mode 100644 airbyte-integrations/connectors/source-my-hours/source_my_hours/run.py create mode 100644 airbyte-integrations/connectors/source-n8n/source_n8n/run.py create mode 100644 airbyte-integrations/connectors/source-nasa/source_nasa/run.py create mode 100644 airbyte-integrations/connectors/source-netsuite/source_netsuite/run.py create mode 100644 airbyte-integrations/connectors/source-news-api/source_news_api/run.py create mode 100644 airbyte-integrations/connectors/source-newsdata/source_newsdata/run.py create mode 100644 airbyte-integrations/connectors/source-notion/source_notion/run.py create mode 100644 airbyte-integrations/connectors/source-nytimes/source_nytimes/run.py create mode 100644 airbyte-integrations/connectors/source-okta/source_okta/run.py create mode 100644 airbyte-integrations/connectors/source-omnisend/source_omnisend/run.py create mode 100644 airbyte-integrations/connectors/source-onesignal/source_onesignal/run.py create mode 100644 airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/run.py create mode 100644 airbyte-integrations/connectors/source-openweather/source_openweather/run.py create mode 100644 airbyte-integrations/connectors/source-opsgenie/source_opsgenie/run.py create mode 100644 airbyte-integrations/connectors/source-orb/source_orb/run.py create mode 100644 airbyte-integrations/connectors/source-orbit/source_orbit/run.py create mode 100644 airbyte-integrations/connectors/source-oura/source_oura/run.py create mode 100644 airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/run.py create mode 100644 airbyte-integrations/connectors/source-outreach/source_outreach/run.py create mode 100644 airbyte-integrations/connectors/source-pagerduty/source_pagerduty/run.py create mode 100644 airbyte-integrations/connectors/source-pardot/source_pardot/run.py create mode 100644 airbyte-integrations/connectors/source-partnerstack/source_partnerstack/run.py create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/run.py create mode 100644 airbyte-integrations/connectors/source-pendo/source_pendo/run.py create mode 100644 airbyte-integrations/connectors/source-persistiq/source_persistiq/run.py create mode 100644 airbyte-integrations/connectors/source-pexels-api/source_pexels_api/run.py create mode 100644 airbyte-integrations/connectors/source-pinterest/source_pinterest/run.py create mode 100644 airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/run.py create mode 100644 airbyte-integrations/connectors/source-plaid/source_plaid/run.py create mode 100644 airbyte-integrations/connectors/source-plausible/source_plausible/run.py create mode 100644 airbyte-integrations/connectors/source-pocket/source_pocket/run.py create mode 100644 airbyte-integrations/connectors/source-pokeapi/source_pokeapi/run.py create mode 100644 airbyte-integrations/connectors/source-polygon-stock-api/source_polygon_stock_api/run.py create mode 100644 airbyte-integrations/connectors/source-posthog/source_posthog/run.py create mode 100644 airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/run.py create mode 100644 airbyte-integrations/connectors/source-prestashop/source_prestashop/run.py create mode 100644 airbyte-integrations/connectors/source-primetric/source_primetric/run.py create mode 100644 airbyte-integrations/connectors/source-public-apis/source_public_apis/run.py create mode 100644 airbyte-integrations/connectors/source-punk-api/source_punk_api/run.py create mode 100644 airbyte-integrations/connectors/source-pypi/source_pypi/run.py create mode 100644 airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/run.py create mode 100644 airbyte-integrations/connectors/source-qonto/source_qonto/run.py create mode 100644 airbyte-integrations/connectors/source-qualaroo/source_qualaroo/run.py create mode 100644 airbyte-integrations/connectors/source-quickbooks/source_quickbooks/run.py create mode 100644 airbyte-integrations/connectors/source-railz/source_railz/run.py create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/run.py create mode 100644 airbyte-integrations/connectors/source-recharge/source_recharge/run.py create mode 100644 airbyte-integrations/connectors/source-recreation/source_recreation/run.py create mode 100644 airbyte-integrations/connectors/source-recruitee/source_recruitee/run.py create mode 100644 airbyte-integrations/connectors/source-recurly/source_recurly/run.py create mode 100644 airbyte-integrations/connectors/source-reply-io/source_reply_io/run.py create mode 100644 airbyte-integrations/connectors/source-retently/source_retently/run.py create mode 100644 airbyte-integrations/connectors/source-ringcentral/source_ringcentral/run.py create mode 100644 airbyte-integrations/connectors/source-rki-covid/source_rki_covid/run.py create mode 100644 airbyte-integrations/connectors/source-rocket-chat/source_rocket_chat/run.py create mode 100644 airbyte-integrations/connectors/source-rss/source_rss/run.py create mode 100644 airbyte-integrations/connectors/source-salesloft/source_salesloft/run.py create mode 100644 airbyte-integrations/connectors/source-sap-fieldglass/source_sap_fieldglass/run.py create mode 100644 airbyte-integrations/connectors/source-scaffold-source-http/source_scaffold_source_http/run.py create mode 100644 airbyte-integrations/connectors/source-scaffold-source-python/source_scaffold_source_python/run.py create mode 100644 airbyte-integrations/connectors/source-search-metrics/source_search_metrics/run.py create mode 100644 airbyte-integrations/connectors/source-secoda/source_secoda/run.py create mode 100644 airbyte-integrations/connectors/source-sendgrid/source_sendgrid/run.py create mode 100644 airbyte-integrations/connectors/source-sendinblue/source_sendinblue/run.py create mode 100644 airbyte-integrations/connectors/source-senseforce/source_senseforce/run.py create mode 100644 airbyte-integrations/connectors/source-sentry/source_sentry/run.py create mode 100644 airbyte-integrations/connectors/source-serpstat/source_serpstat/run.py create mode 100644 airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/run.py create mode 100644 airbyte-integrations/connectors/source-shortio/source_shortio/run.py create mode 100644 airbyte-integrations/connectors/source-smaily/source_smaily/run.py create mode 100644 airbyte-integrations/connectors/source-smartengage/source_smartengage/run.py create mode 100644 airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/run.py create mode 100644 airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/run.py create mode 100644 airbyte-integrations/connectors/source-spacex-api/source_spacex_api/run.py create mode 100644 airbyte-integrations/connectors/source-square/source_square/run.py create mode 100644 airbyte-integrations/connectors/source-statuspage/source_statuspage/run.py create mode 100644 airbyte-integrations/connectors/source-strava/source_strava/run.py create mode 100644 airbyte-integrations/connectors/source-survey-sparrow/source_survey_sparrow/run.py create mode 100644 airbyte-integrations/connectors/source-surveycto/source_surveycto/run.py create mode 100644 airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/run.py create mode 100644 airbyte-integrations/connectors/source-talkdesk-explore/source_talkdesk_explore/run.py create mode 100644 airbyte-integrations/connectors/source-tempo/source_tempo/run.py create mode 100644 airbyte-integrations/connectors/source-the-guardian-api/source_the_guardian_api/run.py create mode 100644 airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/run.py create mode 100644 airbyte-integrations/connectors/source-timely/source_timely/run.py create mode 100644 airbyte-integrations/connectors/source-tmdb/source_tmdb/run.py create mode 100644 airbyte-integrations/connectors/source-todoist/source_todoist/run.py create mode 100644 airbyte-integrations/connectors/source-toggl/source_toggl/run.py create mode 100644 airbyte-integrations/connectors/source-tplcentral/source_tplcentral/run.py create mode 100644 airbyte-integrations/connectors/source-trello/source_trello/run.py create mode 100644 airbyte-integrations/connectors/source-trustpilot/source_trustpilot/run.py create mode 100644 airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/run.py create mode 100644 airbyte-integrations/connectors/source-twilio-taskrouter/source_twilio_taskrouter/run.py create mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/run.py create mode 100644 airbyte-integrations/connectors/source-twitter/source_twitter/run.py create mode 100644 airbyte-integrations/connectors/source-tyntec-sms/source_tyntec_sms/run.py create mode 100644 airbyte-integrations/connectors/source-unleash/source_unleash/run.py create mode 100644 airbyte-integrations/connectors/source-us-census/source_us_census/run.py create mode 100644 airbyte-integrations/connectors/source-vantage/source_vantage/run.py create mode 100644 airbyte-integrations/connectors/source-visma-economic/source_visma_economic/run.py create mode 100644 airbyte-integrations/connectors/source-vitally/source_vitally/run.py create mode 100644 airbyte-integrations/connectors/source-waiteraid/source_waiteraid/run.py create mode 100644 airbyte-integrations/connectors/source-weatherstack/source_weatherstack/run.py create mode 100644 airbyte-integrations/connectors/source-webflow/source_webflow/run.py create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/run.py create mode 100755 airbyte-integrations/connectors/source-wikipedia-pageviews/source_wikipedia_pageviews/run.py create mode 100644 airbyte-integrations/connectors/source-woocommerce/source_woocommerce/run.py create mode 100644 airbyte-integrations/connectors/source-workable/source_workable/run.py create mode 100644 airbyte-integrations/connectors/source-workramp/source_workramp/run.py create mode 100644 airbyte-integrations/connectors/source-wrike/source_wrike/run.py create mode 100644 airbyte-integrations/connectors/source-xkcd/source_xkcd/run.py create mode 100644 airbyte-integrations/connectors/source-yahoo-finance-price/source_yahoo_finance_price/run.py create mode 100644 airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/run.py create mode 100644 airbyte-integrations/connectors/source-yotpo/source_yotpo/run.py create mode 100644 airbyte-integrations/connectors/source-younium/source_younium/run.py create mode 100644 airbyte-integrations/connectors/source-youtube-analytics/source_youtube_analytics/run.py create mode 100644 airbyte-integrations/connectors/source-zapier-supported-storage/source_zapier_supported_storage/run.py create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/run.py create mode 100644 airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/run.py create mode 100644 airbyte-integrations/connectors/source-zenefits/source_zenefits/run.py create mode 100644 airbyte-integrations/connectors/source-zenloop/source_zenloop/run.py create mode 100644 airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/run.py create mode 100644 airbyte-integrations/connectors/source-zoom/source_zoom/run.py create mode 100644 airbyte-integrations/connectors/source-zuora/source_zuora/run.py diff --git a/airbyte-integrations/connectors/source-activecampaign/main.py b/airbyte-integrations/connectors/source-activecampaign/main.py index 289325e3c27e..52ba47917495 100644 --- a/airbyte-integrations/connectors/source-activecampaign/main.py +++ b/airbyte-integrations/connectors/source-activecampaign/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_activecampaign import SourceActivecampaign +from source_activecampaign.run import run if __name__ == "__main__": - source = SourceActivecampaign() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-activecampaign/setup.py b/airbyte-integrations/connectors/source-activecampaign/setup.py index d539a3c2757c..3c64964fcef1 100644 --- a/airbyte-integrations/connectors/source-activecampaign/setup.py +++ b/airbyte-integrations/connectors/source-activecampaign/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-activecampaign=source_activecampaign.run:run", + ], + }, name="source_activecampaign", description="Source implementation for Activecampaign.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/run.py b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/run.py new file mode 100644 index 000000000000..adc9dbcc7e05 --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_activecampaign import SourceActivecampaign + + +def run(): + source = SourceActivecampaign() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-adjust/main.py b/airbyte-integrations/connectors/source-adjust/main.py index 1639515b23ae..06eebde5d55c 100644 --- a/airbyte-integrations/connectors/source-adjust/main.py +++ b/airbyte-integrations/connectors/source-adjust/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_adjust import SourceAdjust +from source_adjust.run import run if __name__ == "__main__": - source = SourceAdjust() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-adjust/setup.py b/airbyte-integrations/connectors/source-adjust/setup.py index 99691631d22c..7195646cdfce 100644 --- a/airbyte-integrations/connectors/source-adjust/setup.py +++ b/airbyte-integrations/connectors/source-adjust/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-adjust=source_adjust.run:run", + ], + }, name="source_adjust", description="Source implementation for Adjust.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-adjust/source_adjust/run.py b/airbyte-integrations/connectors/source-adjust/source_adjust/run.py new file mode 100644 index 000000000000..db3689a7552d --- /dev/null +++ b/airbyte-integrations/connectors/source-adjust/source_adjust/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_adjust import SourceAdjust + + +def run(): + source = SourceAdjust() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-aha/main.py b/airbyte-integrations/connectors/source-aha/main.py index 79ed7a087f63..b07aafa3db46 100644 --- a/airbyte-integrations/connectors/source-aha/main.py +++ b/airbyte-integrations/connectors/source-aha/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_aha import SourceAha +from source_aha.run import run if __name__ == "__main__": - source = SourceAha() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-aha/setup.py b/airbyte-integrations/connectors/source-aha/setup.py index 30789cbe1a0a..bbcedc901f75 100644 --- a/airbyte-integrations/connectors/source-aha/setup.py +++ b/airbyte-integrations/connectors/source-aha/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-aha=source_aha.run:run", + ], + }, name="source_aha", description="Source implementation for Aha.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-aha/source_aha/run.py b/airbyte-integrations/connectors/source-aha/source_aha/run.py new file mode 100644 index 000000000000..7a67cc7e710c --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/source_aha/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_aha import SourceAha + + +def run(): + source = SourceAha() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-aircall/main.py b/airbyte-integrations/connectors/source-aircall/main.py index ff8cceaf862c..3a1a1acff9b2 100644 --- a/airbyte-integrations/connectors/source-aircall/main.py +++ b/airbyte-integrations/connectors/source-aircall/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_aircall import SourceAircall +from source_aircall.run import run if __name__ == "__main__": - source = SourceAircall() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-aircall/setup.py b/airbyte-integrations/connectors/source-aircall/setup.py index 25b830a1e3cc..8453ded69fca 100644 --- a/airbyte-integrations/connectors/source-aircall/setup.py +++ b/airbyte-integrations/connectors/source-aircall/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-aircall=source_aircall.run:run", + ], + }, name="source_aircall", description="Source implementation for Aircall.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-aircall/source_aircall/run.py b/airbyte-integrations/connectors/source-aircall/source_aircall/run.py new file mode 100644 index 000000000000..b6d0e6bb463c --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/source_aircall/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_aircall import SourceAircall + + +def run(): + source = SourceAircall() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-airtable/main.py b/airbyte-integrations/connectors/source-airtable/main.py index 61aedaa8b88d..170d6caf75b1 100644 --- a/airbyte-integrations/connectors/source-airtable/main.py +++ b/airbyte-integrations/connectors/source-airtable/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_airtable import SourceAirtable +from source_airtable.run import run if __name__ == "__main__": - source = SourceAirtable() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-airtable/setup.py b/airbyte-integrations/connectors/source-airtable/setup.py index 2c294e0b0dfb..9cae02970539 100644 --- a/airbyte-integrations/connectors/source-airtable/setup.py +++ b/airbyte-integrations/connectors/source-airtable/setup.py @@ -16,6 +16,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-airtable=source_airtable.run:run", + ], + }, name="source_airtable", description="Source implementation for Airtable.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/run.py b/airbyte-integrations/connectors/source-airtable/source_airtable/run.py new file mode 100644 index 000000000000..e993fd6eaa7d --- /dev/null +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_airtable import SourceAirtable + + +def run(): + source = SourceAirtable() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-alpha-vantage/main.py b/airbyte-integrations/connectors/source-alpha-vantage/main.py index 422447d19ea6..dcccfe7a535c 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/main.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_alpha_vantage import SourceAlphaVantage +from source_alpha_vantage.run import run if __name__ == "__main__": - source = SourceAlphaVantage() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-alpha-vantage/setup.py b/airbyte-integrations/connectors/source-alpha-vantage/setup.py index bf4ebe78c025..22bb9790ab51 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/setup.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-alpha-vantage=source_alpha_vantage.run:run", + ], + }, name="source_alpha_vantage", description="Source implementation for Alpha Vantage.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/run.py b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/run.py new file mode 100644 index 000000000000..fe5a71ac01fb --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_alpha_vantage import SourceAlphaVantage + + +def run(): + source = SourceAlphaVantage() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-amazon-ads/main.py b/airbyte-integrations/connectors/source-amazon-ads/main.py index 1c292d29e4ca..30a0b6957860 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/main.py +++ b/airbyte-integrations/connectors/source-amazon-ads/main.py @@ -2,14 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_amazon_ads import SourceAmazonAds -from source_amazon_ads.config_migrations import MigrateStartDate +from source_amazon_ads.run import run if __name__ == "__main__": - source = SourceAmazonAds() - MigrateStartDate.migrate(sys.argv[1:], source) - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-amazon-ads/setup.py b/airbyte-integrations/connectors/source-amazon-ads/setup.py index 7d612fffeaaa..18026e193950 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/setup.py +++ b/airbyte-integrations/connectors/source-amazon-ads/setup.py @@ -17,6 +17,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-amazon-ads=source_amazon_ads.run:run", + ], + }, name="source_amazon_ads", description="Source implementation for Amazon Ads.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py new file mode 100644 index 000000000000..a8012240de66 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_amazon_ads import SourceAmazonAds +from source_amazon_ads.config_migrations import MigrateStartDate + + +def run(): + source = SourceAmazonAds() + MigrateStartDate.migrate(sys.argv[1:], source) + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/main.py b/airbyte-integrations/connectors/source-amazon-seller-partner/main.py index d53252191baf..ee7f33aa3ce5 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/main.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/main.py @@ -2,15 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_amazon_seller_partner import SourceAmazonSellerPartner -from source_amazon_seller_partner.config_migrations import MigrateAccountType, MigrateReportOptions +from source_amazon_seller_partner.run import run if __name__ == "__main__": - source = SourceAmazonSellerPartner() - MigrateAccountType.migrate(sys.argv[1:], source) - MigrateReportOptions.migrate(sys.argv[1:], source) - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py b/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py index 8a171d3ee035..2b9f48ca5e81 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py @@ -10,6 +10,11 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock"] setup( + entry_points={ + "console_scripts": [ + "source-amazon-seller-partner=source_amazon_seller_partner.run:run", + ], + }, name="source_amazon_seller_partner", description="Source implementation for Amazon Seller Partner.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/run.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/run.py new file mode 100644 index 000000000000..538cf70c8afc --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/run.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_amazon_seller_partner import SourceAmazonSellerPartner +from source_amazon_seller_partner.config_migrations import MigrateAccountType, MigrateReportOptions + + +def run(): + source = SourceAmazonSellerPartner() + MigrateAccountType.migrate(sys.argv[1:], source) + MigrateReportOptions.migrate(sys.argv[1:], source) + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-amazon-sqs/main.py b/airbyte-integrations/connectors/source-amazon-sqs/main.py index bbf86753b1de..3e218a144f8f 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/main.py +++ b/airbyte-integrations/connectors/source-amazon-sqs/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_amazon_sqs import SourceAmazonSqs +from source_amazon_sqs.run import run if __name__ == "__main__": - source = SourceAmazonSqs() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-amazon-sqs/setup.py b/airbyte-integrations/connectors/source-amazon-sqs/setup.py index 3ca84d0f1041..12414e180650 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/setup.py +++ b/airbyte-integrations/connectors/source-amazon-sqs/setup.py @@ -10,6 +10,11 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1", "moto[sqs, iam]"] setup( + entry_points={ + "console_scripts": [ + "source-amazon-sqs=source_amazon_sqs.run:run", + ], + }, name="source_amazon_sqs", description="Source implementation for Amazon Sqs.", author="Alasdair Brown", diff --git a/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/run.py b/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/run.py new file mode 100644 index 000000000000..428858388b63 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_amazon_sqs import SourceAmazonSqs + + +def run(): + source = SourceAmazonSqs() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-amplitude/main.py b/airbyte-integrations/connectors/source-amplitude/main.py index 7bf14f9904f8..14500e9c73e6 100644 --- a/airbyte-integrations/connectors/source-amplitude/main.py +++ b/airbyte-integrations/connectors/source-amplitude/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_amplitude import SourceAmplitude +from source_amplitude.run import run if __name__ == "__main__": - source = SourceAmplitude() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-amplitude/setup.py b/airbyte-integrations/connectors/source-amplitude/setup.py index 29cccb75e13f..2f3809b7ffe9 100644 --- a/airbyte-integrations/connectors/source-amplitude/setup.py +++ b/airbyte-integrations/connectors/source-amplitude/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-amplitude=source_amplitude.run:run", + ], + }, name="source_amplitude", description="Source implementation for Amplitude.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/run.py b/airbyte-integrations/connectors/source-amplitude/source_amplitude/run.py new file mode 100644 index 000000000000..3649e5d9b311 --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_amplitude import SourceAmplitude + + +def run(): + source = SourceAmplitude() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-apify-dataset/setup.py b/airbyte-integrations/connectors/source-apify-dataset/setup.py index eb3a7db40ab2..994bf4b0f951 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/setup.py +++ b/airbyte-integrations/connectors/source-apify-dataset/setup.py @@ -16,7 +16,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-appfollow/main.py b/airbyte-integrations/connectors/source-appfollow/main.py index a4cd4bd63690..79dea68512c5 100644 --- a/airbyte-integrations/connectors/source-appfollow/main.py +++ b/airbyte-integrations/connectors/source-appfollow/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_appfollow import SourceAppfollow +from source_appfollow.run import run if __name__ == "__main__": - source = SourceAppfollow() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-appfollow/setup.py b/airbyte-integrations/connectors/source-appfollow/setup.py index c808dd5682e9..73e20b4ae8b1 100644 --- a/airbyte-integrations/connectors/source-appfollow/setup.py +++ b/airbyte-integrations/connectors/source-appfollow/setup.py @@ -12,13 +12,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", "requests_mock~=1.9"] setup( + entry_points={ + "console_scripts": [ + "source-appfollow=source_appfollow.run:run", + ], + }, name="source_appfollow", description="Source implementation for Appfollow.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-appfollow/source_appfollow/run.py b/airbyte-integrations/connectors/source-appfollow/source_appfollow/run.py new file mode 100644 index 000000000000..019ce80f3ba5 --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/source_appfollow/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_appfollow import SourceAppfollow + + +def run(): + source = SourceAppfollow() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-apple-search-ads/main.py b/airbyte-integrations/connectors/source-apple-search-ads/main.py index 8998f6bd4388..df8b0f70775c 100644 --- a/airbyte-integrations/connectors/source-apple-search-ads/main.py +++ b/airbyte-integrations/connectors/source-apple-search-ads/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_apple_search_ads import SourceAppleSearchAds +from source_apple_search_ads.run import run if __name__ == "__main__": - source = SourceAppleSearchAds() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-apple-search-ads/setup.py b/airbyte-integrations/connectors/source-apple-search-ads/setup.py index a66de217e0fd..c70cfcb0168b 100644 --- a/airbyte-integrations/connectors/source-apple-search-ads/setup.py +++ b/airbyte-integrations/connectors/source-apple-search-ads/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-apple-search-ads=source_apple_search_ads.run:run", + ], + }, name="source_apple_search_ads", description="Source implementation for Apple Search Ads.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-apple-search-ads/source_apple_search_ads/run.py b/airbyte-integrations/connectors/source-apple-search-ads/source_apple_search_ads/run.py new file mode 100644 index 000000000000..dc2def6147ad --- /dev/null +++ b/airbyte-integrations/connectors/source-apple-search-ads/source_apple_search_ads/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_apple_search_ads import SourceAppleSearchAds + + +def run(): + source = SourceAppleSearchAds() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-appsflyer/main.py b/airbyte-integrations/connectors/source-appsflyer/main.py index 1dee48f3f0ab..ebf2655b2ec4 100644 --- a/airbyte-integrations/connectors/source-appsflyer/main.py +++ b/airbyte-integrations/connectors/source-appsflyer/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_appsflyer import SourceAppsflyer +from source_appsflyer.run import run if __name__ == "__main__": - source = SourceAppsflyer() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-appsflyer/setup.py b/airbyte-integrations/connectors/source-appsflyer/setup.py index ac74ebcabfc9..cabee5977060 100644 --- a/airbyte-integrations/connectors/source-appsflyer/setup.py +++ b/airbyte-integrations/connectors/source-appsflyer/setup.py @@ -14,6 +14,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-appsflyer=source_appsflyer.run:run", + ], + }, name="source_appsflyer", description="Source implementation for Appsflyer.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-appsflyer/source_appsflyer/run.py b/airbyte-integrations/connectors/source-appsflyer/source_appsflyer/run.py new file mode 100644 index 000000000000..40c1e2feb038 --- /dev/null +++ b/airbyte-integrations/connectors/source-appsflyer/source_appsflyer/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_appsflyer import SourceAppsflyer + + +def run(): + source = SourceAppsflyer() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-appstore-singer/main.py b/airbyte-integrations/connectors/source-appstore-singer/main.py index 5e0f99d007be..34e585afeeed 100644 --- a/airbyte-integrations/connectors/source-appstore-singer/main.py +++ b/airbyte-integrations/connectors/source-appstore-singer/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_appstore_singer import SourceAppstoreSinger +from source_appstore_singer.run import run if __name__ == "__main__": - source = SourceAppstoreSinger() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-appstore-singer/setup.py b/airbyte-integrations/connectors/source-appstore-singer/setup.py index b6ecccc99a03..d85ef5d07f43 100644 --- a/airbyte-integrations/connectors/source-appstore-singer/setup.py +++ b/airbyte-integrations/connectors/source-appstore-singer/setup.py @@ -19,6 +19,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-appstore-singer=source_appstore_singer.run:run", + ], + }, name="source_appstore_singer", description="Source implementation for Appstore, built on the Singer tap implementation.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-appstore-singer/source_appstore_singer/run.py b/airbyte-integrations/connectors/source-appstore-singer/source_appstore_singer/run.py new file mode 100644 index 000000000000..ef9f845e8d81 --- /dev/null +++ b/airbyte-integrations/connectors/source-appstore-singer/source_appstore_singer/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_appstore_singer import SourceAppstoreSinger + + +def run(): + source = SourceAppstoreSinger() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-asana/main.py b/airbyte-integrations/connectors/source-asana/main.py index ac64981a1e3e..5fde4e3c72d6 100644 --- a/airbyte-integrations/connectors/source-asana/main.py +++ b/airbyte-integrations/connectors/source-asana/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_asana import SourceAsana +from source_asana.run import run if __name__ == "__main__": - source = SourceAsana() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-asana/setup.py b/airbyte-integrations/connectors/source-asana/setup.py index dda6ee977db1..7d19a10aad8a 100644 --- a/airbyte-integrations/connectors/source-asana/setup.py +++ b/airbyte-integrations/connectors/source-asana/setup.py @@ -12,6 +12,11 @@ TEST_REQUIREMENTS = ["pytest-mock~=3.6.1", "pytest~=6.1", "requests-mock~=1.9.3"] setup( + entry_points={ + "console_scripts": [ + "source-asana=source_asana.run:run", + ], + }, name="source_asana", description="Source implementation for Asana.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-asana/source_asana/run.py b/airbyte-integrations/connectors/source-asana/source_asana/run.py new file mode 100644 index 000000000000..cf9dbf1ffa48 --- /dev/null +++ b/airbyte-integrations/connectors/source-asana/source_asana/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_asana import SourceAsana + + +def run(): + source = SourceAsana() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-ashby/main.py b/airbyte-integrations/connectors/source-ashby/main.py index a19039ecb615..96b5bfe556e3 100644 --- a/airbyte-integrations/connectors/source-ashby/main.py +++ b/airbyte-integrations/connectors/source-ashby/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_ashby import SourceAshby +from source_ashby.run import run if __name__ == "__main__": - source = SourceAshby() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-ashby/setup.py b/airbyte-integrations/connectors/source-ashby/setup.py index d4fd781c5f1e..d506c08b0965 100644 --- a/airbyte-integrations/connectors/source-ashby/setup.py +++ b/airbyte-integrations/connectors/source-ashby/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-ashby=source_ashby.run:run", + ], + }, name="source_ashby", description="Source implementation for Ashby.", author="Elliot Trabac", author_email="elliot.trabac1@gmail.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-ashby/source_ashby/run.py b/airbyte-integrations/connectors/source-ashby/source_ashby/run.py new file mode 100644 index 000000000000..2330c6f910b5 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/source_ashby/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_ashby import SourceAshby + + +def run(): + source = SourceAshby() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-auth0/main.py b/airbyte-integrations/connectors/source-auth0/main.py index 3904766c4ec2..f29790fbabd7 100644 --- a/airbyte-integrations/connectors/source-auth0/main.py +++ b/airbyte-integrations/connectors/source-auth0/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_auth0 import SourceAuth0 +from source_auth0.run import run if __name__ == "__main__": - source = SourceAuth0() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-auth0/setup.py b/airbyte-integrations/connectors/source-auth0/setup.py index 29b448c8c0de..ed772442f1e0 100644 --- a/airbyte-integrations/connectors/source-auth0/setup.py +++ b/airbyte-integrations/connectors/source-auth0/setup.py @@ -15,13 +15,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-auth0=source_auth0.run:run", + ], + }, name="source_auth0", description="Source implementation for Auth0.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/run.py b/airbyte-integrations/connectors/source-auth0/source_auth0/run.py new file mode 100644 index 000000000000..bae97c9f5cb4 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/source_auth0/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_auth0 import SourceAuth0 + + +def run(): + source = SourceAuth0() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/main.py b/airbyte-integrations/connectors/source-aws-cloudtrail/main.py index 7ae051b2be22..f2324dfe8812 100644 --- a/airbyte-integrations/connectors/source-aws-cloudtrail/main.py +++ b/airbyte-integrations/connectors/source-aws-cloudtrail/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_aws_cloudtrail import SourceAwsCloudtrail +from source_aws_cloudtrail.run import run if __name__ == "__main__": - source = SourceAwsCloudtrail() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/setup.py b/airbyte-integrations/connectors/source-aws-cloudtrail/setup.py index bda35b60aa1e..77d6bdf20486 100644 --- a/airbyte-integrations/connectors/source-aws-cloudtrail/setup.py +++ b/airbyte-integrations/connectors/source-aws-cloudtrail/setup.py @@ -14,6 +14,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-aws-cloudtrail=source_aws_cloudtrail.run:run", + ], + }, name="source_aws_cloudtrail", description="Source implementation for Aws Cloudtrail.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/source_aws_cloudtrail/run.py b/airbyte-integrations/connectors/source-aws-cloudtrail/source_aws_cloudtrail/run.py new file mode 100644 index 000000000000..576aae749bc2 --- /dev/null +++ b/airbyte-integrations/connectors/source-aws-cloudtrail/source_aws_cloudtrail/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_aws_cloudtrail import SourceAwsCloudtrail + + +def run(): + source = SourceAwsCloudtrail() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/main.py b/airbyte-integrations/connectors/source-azure-blob-storage/main.py index b3361a6556d7..5e798013d9e4 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/main.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/main.py @@ -2,32 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import sys -import traceback -from datetime import datetime - -from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch -from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type -from source_azure_blob_storage import Config, SourceAzureBlobStorage, SourceAzureBlobStorageStreamReader +from source_azure_blob_storage.run import run if __name__ == "__main__": - args = sys.argv[1:] - catalog_path = AirbyteEntrypoint.extract_catalog(args) - try: - source = SourceAzureBlobStorage(SourceAzureBlobStorageStreamReader(), Config, catalog_path) - except Exception: - print( - AirbyteMessage( - type=Type.TRACE, - trace=AirbyteTraceMessage( - type=TraceType.ERROR, - emitted_at=int(datetime.now().timestamp() * 1000), - error=AirbyteErrorTraceMessage( - message="Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance.", - stack_trace=traceback.format_exc(), - ), - ), - ).json() - ) - else: - launch(source, args) + run() diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/setup.py b/airbyte-integrations/connectors/source-azure-blob-storage/setup.py index 4246eb43364c..1dc7c4a275c4 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/setup.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/setup.py @@ -14,6 +14,11 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.2"] setup( + entry_points={ + "console_scripts": [ + "source-azure-blob-storage=source_azure_blob_storage.run:run", + ], + }, name="source_azure_blob_storage", description="Source implementation for Azure Blob Storage.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/run.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/run.py new file mode 100644 index 000000000000..404d919b60f3 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/run.py @@ -0,0 +1,34 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import sys +import traceback +from datetime import datetime + +from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch +from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type +from source_azure_blob_storage import Config, SourceAzureBlobStorage, SourceAzureBlobStorageStreamReader + + +def run(): + args = sys.argv[1:] + catalog_path = AirbyteEntrypoint.extract_catalog(args) + try: + source = SourceAzureBlobStorage(SourceAzureBlobStorageStreamReader(), Config, catalog_path) + except Exception: + print( + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.ERROR, + emitted_at=int(datetime.now().timestamp() * 1000), + error=AirbyteErrorTraceMessage( + message="Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance.", + stack_trace=traceback.format_exc(), + ), + ), + ).json() + ) + else: + launch(source, args) diff --git a/airbyte-integrations/connectors/source-azure-table/main.py b/airbyte-integrations/connectors/source-azure-table/main.py index ffdca7c26cef..0831f8065766 100644 --- a/airbyte-integrations/connectors/source-azure-table/main.py +++ b/airbyte-integrations/connectors/source-azure-table/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_azure_table import SourceAzureTable +from source_azure_table.run import run if __name__ == "__main__": - source = SourceAzureTable() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-azure-table/setup.py b/airbyte-integrations/connectors/source-azure-table/setup.py index a04c790cbb8f..31eceef9c80e 100644 --- a/airbyte-integrations/connectors/source-azure-table/setup.py +++ b/airbyte-integrations/connectors/source-azure-table/setup.py @@ -14,6 +14,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-azure-table=source_azure_table.run:run", + ], + }, name="source_azure_table", description="Source implementation for Azure Table.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-azure-table/source_azure_table/run.py b/airbyte-integrations/connectors/source-azure-table/source_azure_table/run.py new file mode 100644 index 000000000000..b39667cd684a --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-table/source_azure_table/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_azure_table import SourceAzureTable + + +def run(): + source = SourceAzureTable() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-babelforce/main.py b/airbyte-integrations/connectors/source-babelforce/main.py index 33b4c6d35a82..da6273a1dca9 100644 --- a/airbyte-integrations/connectors/source-babelforce/main.py +++ b/airbyte-integrations/connectors/source-babelforce/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_babelforce import SourceBabelforce +from source_babelforce.run import run if __name__ == "__main__": - source = SourceBabelforce() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-babelforce/setup.py b/airbyte-integrations/connectors/source-babelforce/setup.py index 36286f264643..5033ec7b34e2 100644 --- a/airbyte-integrations/connectors/source-babelforce/setup.py +++ b/airbyte-integrations/connectors/source-babelforce/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-babelforce=source_babelforce.run:run", + ], + }, name="source_babelforce", description="Source implementation for Babelforce.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-babelforce/source_babelforce/run.py b/airbyte-integrations/connectors/source-babelforce/source_babelforce/run.py new file mode 100644 index 000000000000..056cf590f326 --- /dev/null +++ b/airbyte-integrations/connectors/source-babelforce/source_babelforce/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_babelforce import SourceBabelforce + + +def run(): + source = SourceBabelforce() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-bamboo-hr/main.py b/airbyte-integrations/connectors/source-bamboo-hr/main.py index 2ebc95cb1256..0118185a67fc 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/main.py +++ b/airbyte-integrations/connectors/source-bamboo-hr/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_bamboo_hr import SourceBambooHr +from source_bamboo_hr.run import run if __name__ == "__main__": - source = SourceBambooHr() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-bamboo-hr/setup.py b/airbyte-integrations/connectors/source-bamboo-hr/setup.py index a81279a9f2dd..52914c4c34d4 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/setup.py +++ b/airbyte-integrations/connectors/source-bamboo-hr/setup.py @@ -14,6 +14,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-bamboo-hr=source_bamboo_hr.run:run", + ], + }, name="source_bamboo_hr", description="Source implementation for Bamboo Hr.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/run.py b/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/run.py new file mode 100644 index 000000000000..d9a04d56a964 --- /dev/null +++ b/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_bamboo_hr import SourceBambooHr + + +def run(): + source = SourceBambooHr() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-bigcommerce/main.py b/airbyte-integrations/connectors/source-bigcommerce/main.py index 2d28000deab0..7830bf519a65 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/main.py +++ b/airbyte-integrations/connectors/source-bigcommerce/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_bigcommerce import SourceBigcommerce +from source_bigcommerce.run import run if __name__ == "__main__": - source = SourceBigcommerce() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-bigcommerce/setup.py b/airbyte-integrations/connectors/source-bigcommerce/setup.py index 7189d313888f..43cc76692ad7 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/setup.py +++ b/airbyte-integrations/connectors/source-bigcommerce/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-bigcommerce=source_bigcommerce.run:run", + ], + }, name="source_bigcommerce", description="Source implementation for Bigcommerce.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/run.py b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/run.py new file mode 100644 index 000000000000..7e5234a1b2b1 --- /dev/null +++ b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_bigcommerce import SourceBigcommerce + + +def run(): + source = SourceBigcommerce() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-bing-ads/main.py b/airbyte-integrations/connectors/source-bing-ads/main.py index 11548ad408a8..c05297b01ad8 100644 --- a/airbyte-integrations/connectors/source-bing-ads/main.py +++ b/airbyte-integrations/connectors/source-bing-ads/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_bing_ads import SourceBingAds +from source_bing_ads.run import run if __name__ == "__main__": - source = SourceBingAds() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-bing-ads/setup.py b/airbyte-integrations/connectors/source-bing-ads/setup.py index e586d0ea27ae..68f92f541a5f 100644 --- a/airbyte-integrations/connectors/source-bing-ads/setup.py +++ b/airbyte-integrations/connectors/source-bing-ads/setup.py @@ -15,6 +15,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-bing-ads=source_bing_ads.run:run", + ], + }, name="source_bing_ads", description="Source implementation for Bing Ads.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/run.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/run.py new file mode 100644 index 000000000000..3dd73c2fec5e --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_bing_ads import SourceBingAds + + +def run(): + source = SourceBingAds() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-braintree/main.py b/airbyte-integrations/connectors/source-braintree/main.py index 3b3c6039f9fe..d4ae7bec5223 100644 --- a/airbyte-integrations/connectors/source-braintree/main.py +++ b/airbyte-integrations/connectors/source-braintree/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_braintree import SourceBraintree +from source_braintree.run import run if __name__ == "__main__": - source = SourceBraintree() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-braintree/setup.py b/airbyte-integrations/connectors/source-braintree/setup.py index 0a6f5d53752e..cd584b74e699 100644 --- a/airbyte-integrations/connectors/source-braintree/setup.py +++ b/airbyte-integrations/connectors/source-braintree/setup.py @@ -13,13 +13,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-braintree=source_braintree.run:run", + ], + }, name="source_braintree_no_code", description="Source implementation for Braintree No Code.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-braintree/source_braintree/run.py b/airbyte-integrations/connectors/source-braintree/source_braintree/run.py new file mode 100644 index 000000000000..df776d3952c6 --- /dev/null +++ b/airbyte-integrations/connectors/source-braintree/source_braintree/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_braintree import SourceBraintree + + +def run(): + source = SourceBraintree() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-braze/main.py b/airbyte-integrations/connectors/source-braze/main.py index 15453641a7f0..723116b28098 100644 --- a/airbyte-integrations/connectors/source-braze/main.py +++ b/airbyte-integrations/connectors/source-braze/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_braze import SourceBraze +from source_braze.run import run if __name__ == "__main__": - source = SourceBraze() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-braze/setup.py b/airbyte-integrations/connectors/source-braze/setup.py index aade45358b41..43f778382f75 100644 --- a/airbyte-integrations/connectors/source-braze/setup.py +++ b/airbyte-integrations/connectors/source-braze/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-braze=source_braze.run:run", + ], + }, name="source_braze", description="Source implementation for Braze.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-braze/source_braze/run.py b/airbyte-integrations/connectors/source-braze/source_braze/run.py new file mode 100644 index 000000000000..645b7a31df24 --- /dev/null +++ b/airbyte-integrations/connectors/source-braze/source_braze/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_braze import SourceBraze + + +def run(): + source = SourceBraze() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-breezometer/main.py b/airbyte-integrations/connectors/source-breezometer/main.py index e14371ac83bb..3fc185eec93c 100644 --- a/airbyte-integrations/connectors/source-breezometer/main.py +++ b/airbyte-integrations/connectors/source-breezometer/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_breezometer import SourceBreezometer +from source_breezometer.run import run if __name__ == "__main__": - source = SourceBreezometer() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-breezometer/setup.py b/airbyte-integrations/connectors/source-breezometer/setup.py index 61bd2c525610..fc9d1e85cd20 100644 --- a/airbyte-integrations/connectors/source-breezometer/setup.py +++ b/airbyte-integrations/connectors/source-breezometer/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-breezometer=source_breezometer.run:run", + ], + }, name="source_breezometer", description="Source implementation for Breezometer.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-breezometer/source_breezometer/run.py b/airbyte-integrations/connectors/source-breezometer/source_breezometer/run.py new file mode 100644 index 000000000000..6855613e5586 --- /dev/null +++ b/airbyte-integrations/connectors/source-breezometer/source_breezometer/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_breezometer import SourceBreezometer + + +def run(): + source = SourceBreezometer() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-callrail/main.py b/airbyte-integrations/connectors/source-callrail/main.py index cb3607679473..d5651af4f615 100644 --- a/airbyte-integrations/connectors/source-callrail/main.py +++ b/airbyte-integrations/connectors/source-callrail/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_callrail import SourceCallrail +from source_callrail.run import run if __name__ == "__main__": - source = SourceCallrail() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-callrail/setup.py b/airbyte-integrations/connectors/source-callrail/setup.py index c744bbc957ff..e6d0d00d5d46 100644 --- a/airbyte-integrations/connectors/source-callrail/setup.py +++ b/airbyte-integrations/connectors/source-callrail/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-callrail=source_callrail.run:run", + ], + }, name="source_callrail", description="Source implementation for Callrail.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-callrail/source_callrail/run.py b/airbyte-integrations/connectors/source-callrail/source_callrail/run.py new file mode 100644 index 000000000000..d271cf7a502b --- /dev/null +++ b/airbyte-integrations/connectors/source-callrail/source_callrail/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_callrail import SourceCallrail + + +def run(): + source = SourceCallrail() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-captain-data/main.py b/airbyte-integrations/connectors/source-captain-data/main.py index 765d967fad15..50a0dfd7944d 100644 --- a/airbyte-integrations/connectors/source-captain-data/main.py +++ b/airbyte-integrations/connectors/source-captain-data/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_captain_data import SourceCaptainData +from source_captain_data.run import run if __name__ == "__main__": - source = SourceCaptainData() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-captain-data/setup.py b/airbyte-integrations/connectors/source-captain-data/setup.py index f6121791ddd8..cf4b2f7f5fc7 100644 --- a/airbyte-integrations/connectors/source-captain-data/setup.py +++ b/airbyte-integrations/connectors/source-captain-data/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-captain-data=source_captain_data.run:run", + ], + }, name="source_captain_data", description="Source implementation for Captain Data.", author="Elliot Trabac", author_email="elliot.trabac1@gmail.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/run.py b/airbyte-integrations/connectors/source-captain-data/source_captain_data/run.py new file mode 100644 index 000000000000..5b25fd1be5a4 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_captain_data import SourceCaptainData + + +def run(): + source = SourceCaptainData() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-cart/main.py b/airbyte-integrations/connectors/source-cart/main.py index b294ae4e2c15..c7f69c914848 100644 --- a/airbyte-integrations/connectors/source-cart/main.py +++ b/airbyte-integrations/connectors/source-cart/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_cart import SourceCart +from source_cart.run import run if __name__ == "__main__": - source = SourceCart() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-cart/setup.py b/airbyte-integrations/connectors/source-cart/setup.py index 69df33c757d3..fe389ea16d47 100644 --- a/airbyte-integrations/connectors/source-cart/setup.py +++ b/airbyte-integrations/connectors/source-cart/setup.py @@ -16,6 +16,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-cart=source_cart.run:run", + ], + }, name="source_cart", description="Source implementation for Cart.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-cart/source_cart/run.py b/airbyte-integrations/connectors/source-cart/source_cart/run.py new file mode 100644 index 000000000000..7f639ab6c694 --- /dev/null +++ b/airbyte-integrations/connectors/source-cart/source_cart/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_cart import SourceCart + + +def run(): + source = SourceCart() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-chargebee/main.py b/airbyte-integrations/connectors/source-chargebee/main.py index 946cf215df2d..351ea1590b35 100644 --- a/airbyte-integrations/connectors/source-chargebee/main.py +++ b/airbyte-integrations/connectors/source-chargebee/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_chargebee import SourceChargebee +from source_chargebee.run import run if __name__ == "__main__": - source = SourceChargebee() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-chargebee/setup.py b/airbyte-integrations/connectors/source-chargebee/setup.py index 5c3b625e0b4c..f201051e3f11 100644 --- a/airbyte-integrations/connectors/source-chargebee/setup.py +++ b/airbyte-integrations/connectors/source-chargebee/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-chargebee=source_chargebee.run:run", + ], + }, name="source_chargebee", description="Source implementation for Chargebee.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/run.py b/airbyte-integrations/connectors/source-chargebee/source_chargebee/run.py new file mode 100644 index 000000000000..5c0b427da197 --- /dev/null +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_chargebee import SourceChargebee + + +def run(): + source = SourceChargebee() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-chargify/main.py b/airbyte-integrations/connectors/source-chargify/main.py index 1a4568ff615b..44e0f1156271 100644 --- a/airbyte-integrations/connectors/source-chargify/main.py +++ b/airbyte-integrations/connectors/source-chargify/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_chargify import SourceChargify +from source_chargify.run import run if __name__ == "__main__": - source = SourceChargify() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-chargify/setup.py b/airbyte-integrations/connectors/source-chargify/setup.py index 521c61646023..deda5abcc0b0 100644 --- a/airbyte-integrations/connectors/source-chargify/setup.py +++ b/airbyte-integrations/connectors/source-chargify/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-chargify=source_chargify.run:run", + ], + }, name="source_chargify", description="Source implementation for Chargify.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-chargify/source_chargify/run.py b/airbyte-integrations/connectors/source-chargify/source_chargify/run.py new file mode 100644 index 000000000000..88f4450c9bcf --- /dev/null +++ b/airbyte-integrations/connectors/source-chargify/source_chargify/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_chargify import SourceChargify + + +def run(): + source = SourceChargify() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-chartmogul/main.py b/airbyte-integrations/connectors/source-chartmogul/main.py index 5dc03e8a74a9..bf13dab9878d 100644 --- a/airbyte-integrations/connectors/source-chartmogul/main.py +++ b/airbyte-integrations/connectors/source-chartmogul/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_chartmogul import SourceChartmogul +from source_chartmogul.run import run if __name__ == "__main__": - source = SourceChartmogul() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-chartmogul/setup.py b/airbyte-integrations/connectors/source-chartmogul/setup.py index fa0d73f436c9..57b6c73fbac8 100644 --- a/airbyte-integrations/connectors/source-chartmogul/setup.py +++ b/airbyte-integrations/connectors/source-chartmogul/setup.py @@ -16,6 +16,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-chartmogul=source_chartmogul.run:run", + ], + }, name="source_chartmogul", description="Source implementation for Chartmogul.", author="Titas Skrebe", diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/run.py b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/run.py new file mode 100644 index 000000000000..f8e5bbd30e63 --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_chartmogul import SourceChartmogul + + +def run(): + source = SourceChartmogul() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-clickup-api/main.py b/airbyte-integrations/connectors/source-clickup-api/main.py index 01d656f16c43..76c09b38846f 100644 --- a/airbyte-integrations/connectors/source-clickup-api/main.py +++ b/airbyte-integrations/connectors/source-clickup-api/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_clickup_api import SourceClickupApi +from source_clickup_api.run import run if __name__ == "__main__": - source = SourceClickupApi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-clickup-api/setup.py b/airbyte-integrations/connectors/source-clickup-api/setup.py index 4fb2367a7516..b23f4ed53a1a 100644 --- a/airbyte-integrations/connectors/source-clickup-api/setup.py +++ b/airbyte-integrations/connectors/source-clickup-api/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-clickup-api=source_clickup_api.run:run", + ], + }, name="source_clickup_api", description="Source implementation for Clickup Api.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-clickup-api/source_clickup_api/run.py b/airbyte-integrations/connectors/source-clickup-api/source_clickup_api/run.py new file mode 100644 index 000000000000..3767b66395e9 --- /dev/null +++ b/airbyte-integrations/connectors/source-clickup-api/source_clickup_api/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_clickup_api import SourceClickupApi + + +def run(): + source = SourceClickupApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-clockify/main.py b/airbyte-integrations/connectors/source-clockify/main.py index 9f04d4eb176c..486525f0e293 100644 --- a/airbyte-integrations/connectors/source-clockify/main.py +++ b/airbyte-integrations/connectors/source-clockify/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_clockify import SourceClockify +from source_clockify.run import run if __name__ == "__main__": - source = SourceClockify() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-clockify/setup.py b/airbyte-integrations/connectors/source-clockify/setup.py index bcd38b28c29a..940c87ba74b4 100644 --- a/airbyte-integrations/connectors/source-clockify/setup.py +++ b/airbyte-integrations/connectors/source-clockify/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-clockify=source_clockify.run:run", + ], + }, name="source_clockify", description="Source implementation for Clockify.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/run.py b/airbyte-integrations/connectors/source-clockify/source_clockify/run.py new file mode 100644 index 000000000000..ef88995f79dd --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_clockify import SourceClockify + + +def run(): + source = SourceClockify() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-close-com/main.py b/airbyte-integrations/connectors/source-close-com/main.py index 22787dfbd89e..f80e76315939 100644 --- a/airbyte-integrations/connectors/source-close-com/main.py +++ b/airbyte-integrations/connectors/source-close-com/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_close_com import SourceCloseCom +from source_close_com.run import run if __name__ == "__main__": - source = SourceCloseCom() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-close-com/setup.py b/airbyte-integrations/connectors/source-close-com/setup.py index b9d9aaf53e7d..a67ed8241bb0 100644 --- a/airbyte-integrations/connectors/source-close-com/setup.py +++ b/airbyte-integrations/connectors/source-close-com/setup.py @@ -14,6 +14,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-close-com=source_close_com.run:run", + ], + }, name="source_close_com", description="Source implementation for Close.com.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/run.py b/airbyte-integrations/connectors/source-close-com/source_close_com/run.py new file mode 100644 index 000000000000..eb80c6eab53e --- /dev/null +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_close_com import SourceCloseCom + + +def run(): + source = SourceCloseCom() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-coda/main.py b/airbyte-integrations/connectors/source-coda/main.py index aae501071c7d..dbc36f9d6886 100644 --- a/airbyte-integrations/connectors/source-coda/main.py +++ b/airbyte-integrations/connectors/source-coda/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_coda import SourceCoda +from source_coda.run import run if __name__ == "__main__": - source = SourceCoda() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-coda/setup.py b/airbyte-integrations/connectors/source-coda/setup.py index 446b5807dcec..92e0b6526fe7 100644 --- a/airbyte-integrations/connectors/source-coda/setup.py +++ b/airbyte-integrations/connectors/source-coda/setup.py @@ -15,13 +15,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-coda=source_coda.run:run", + ], + }, name="source_coda", description="Source implementation for Coda.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-coda/source_coda/run.py b/airbyte-integrations/connectors/source-coda/source_coda/run.py new file mode 100644 index 000000000000..0a1547db7b47 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/source_coda/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_coda import SourceCoda + + +def run(): + source = SourceCoda() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-coin-api/main.py b/airbyte-integrations/connectors/source-coin-api/main.py index 44b7d1ff5d1a..0a62eaad6b92 100644 --- a/airbyte-integrations/connectors/source-coin-api/main.py +++ b/airbyte-integrations/connectors/source-coin-api/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_coin_api import SourceCoinApi +from source_coin_api.run import run if __name__ == "__main__": - source = SourceCoinApi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-coin-api/setup.py b/airbyte-integrations/connectors/source-coin-api/setup.py index b4c098be7797..904de0b88661 100644 --- a/airbyte-integrations/connectors/source-coin-api/setup.py +++ b/airbyte-integrations/connectors/source-coin-api/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-coin-api=source_coin_api.run:run", + ], + }, name="source_coin_api", description="Source implementation for Coin Api.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-coin-api/source_coin_api/run.py b/airbyte-integrations/connectors/source-coin-api/source_coin_api/run.py new file mode 100644 index 000000000000..d0d729797ece --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/source_coin_api/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_coin_api import SourceCoinApi + + +def run(): + source = SourceCoinApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-coingecko-coins/main.py b/airbyte-integrations/connectors/source-coingecko-coins/main.py index 6ab339e39a73..0317d9bbf40b 100644 --- a/airbyte-integrations/connectors/source-coingecko-coins/main.py +++ b/airbyte-integrations/connectors/source-coingecko-coins/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_coingecko_coins import SourceCoingeckoCoins +from source_coingecko_coins.run import run if __name__ == "__main__": - source = SourceCoingeckoCoins() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-coingecko-coins/setup.py b/airbyte-integrations/connectors/source-coingecko-coins/setup.py index 39a959dcdc27..efc1c015a4df 100644 --- a/airbyte-integrations/connectors/source-coingecko-coins/setup.py +++ b/airbyte-integrations/connectors/source-coingecko-coins/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-coingecko-coins=source_coingecko_coins.run:run", + ], + }, name="source_coingecko_coins", description="Source implementation for Coingecko Coins.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/run.py b/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/run.py new file mode 100644 index 000000000000..c652fd9af6d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_coingecko_coins import SourceCoingeckoCoins + + +def run(): + source = SourceCoingeckoCoins() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-coinmarketcap/main.py b/airbyte-integrations/connectors/source-coinmarketcap/main.py index e2c19d866514..908e01700613 100644 --- a/airbyte-integrations/connectors/source-coinmarketcap/main.py +++ b/airbyte-integrations/connectors/source-coinmarketcap/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_coinmarketcap import SourceCoinmarketcap +from source_coinmarketcap.run import run if __name__ == "__main__": - source = SourceCoinmarketcap() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-coinmarketcap/setup.py b/airbyte-integrations/connectors/source-coinmarketcap/setup.py index 64fd1b41273b..608c7f92baef 100644 --- a/airbyte-integrations/connectors/source-coinmarketcap/setup.py +++ b/airbyte-integrations/connectors/source-coinmarketcap/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-coinmarketcap=source_coinmarketcap.run:run", + ], + }, name="source_coinmarketcap", description="Source implementation for Coinmarketcap.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-coinmarketcap/source_coinmarketcap/run.py b/airbyte-integrations/connectors/source-coinmarketcap/source_coinmarketcap/run.py new file mode 100644 index 000000000000..523d670c3aa9 --- /dev/null +++ b/airbyte-integrations/connectors/source-coinmarketcap/source_coinmarketcap/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_coinmarketcap import SourceCoinmarketcap + + +def run(): + source = SourceCoinmarketcap() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-commcare/main.py b/airbyte-integrations/connectors/source-commcare/main.py index 362386c57b33..edd438bde5be 100644 --- a/airbyte-integrations/connectors/source-commcare/main.py +++ b/airbyte-integrations/connectors/source-commcare/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_commcare import SourceCommcare +from source_commcare.run import run if __name__ == "__main__": - source = SourceCommcare() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-commcare/setup.py b/airbyte-integrations/connectors/source-commcare/setup.py index fd011eec2b1f..44ba9e72640a 100644 --- a/airbyte-integrations/connectors/source-commcare/setup.py +++ b/airbyte-integrations/connectors/source-commcare/setup.py @@ -19,13 +19,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-commcare=source_commcare.run:run", + ], + }, name="source_commcare", description="Source implementation for Commcare.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-commcare/source_commcare/run.py b/airbyte-integrations/connectors/source-commcare/source_commcare/run.py new file mode 100644 index 000000000000..d42251049d08 --- /dev/null +++ b/airbyte-integrations/connectors/source-commcare/source_commcare/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_commcare import SourceCommcare + + +def run(): + source = SourceCommcare() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-commercetools/main.py b/airbyte-integrations/connectors/source-commercetools/main.py index 32aac6ed9f30..44dd2fb8f952 100644 --- a/airbyte-integrations/connectors/source-commercetools/main.py +++ b/airbyte-integrations/connectors/source-commercetools/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_commercetools import SourceCommercetools +from source_commercetools.run import run if __name__ == "__main__": - source = SourceCommercetools() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-commercetools/setup.py b/airbyte-integrations/connectors/source-commercetools/setup.py index 9622d7ad2c24..386e9738b744 100644 --- a/airbyte-integrations/connectors/source-commercetools/setup.py +++ b/airbyte-integrations/connectors/source-commercetools/setup.py @@ -10,13 +10,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-commercetools=source_commercetools.run:run", + ], + }, name="source_commercetools", description="Source implementation for Commercetools.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/run.py b/airbyte-integrations/connectors/source-commercetools/source_commercetools/run.py new file mode 100644 index 000000000000..0d264787f978 --- /dev/null +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_commercetools import SourceCommercetools + + +def run(): + source = SourceCommercetools() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-configcat/main.py b/airbyte-integrations/connectors/source-configcat/main.py index b398a6dc8c92..9b554d1d713f 100644 --- a/airbyte-integrations/connectors/source-configcat/main.py +++ b/airbyte-integrations/connectors/source-configcat/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_configcat import SourceConfigcat +from source_configcat.run import run if __name__ == "__main__": - source = SourceConfigcat() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-configcat/setup.py b/airbyte-integrations/connectors/source-configcat/setup.py index 0bfbad2e7170..0f65dc21183f 100644 --- a/airbyte-integrations/connectors/source-configcat/setup.py +++ b/airbyte-integrations/connectors/source-configcat/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-configcat=source_configcat.run:run", + ], + }, name="source_configcat", description="Source implementation for Configcat.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-configcat/source_configcat/run.py b/airbyte-integrations/connectors/source-configcat/source_configcat/run.py new file mode 100644 index 000000000000..9167f73fa44c --- /dev/null +++ b/airbyte-integrations/connectors/source-configcat/source_configcat/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_configcat import SourceConfigcat + + +def run(): + source = SourceConfigcat() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-confluence/main.py b/airbyte-integrations/connectors/source-confluence/main.py index 5cf3cd879713..eb38dbbac0f0 100644 --- a/airbyte-integrations/connectors/source-confluence/main.py +++ b/airbyte-integrations/connectors/source-confluence/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_confluence import SourceConfluence +from source_confluence.run import run if __name__ == "__main__": - source = SourceConfluence() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-confluence/setup.py b/airbyte-integrations/connectors/source-confluence/setup.py index aaacdff5bbe3..993c131b35f9 100644 --- a/airbyte-integrations/connectors/source-confluence/setup.py +++ b/airbyte-integrations/connectors/source-confluence/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-confluence=source_confluence.run:run", + ], + }, name="source_confluence", description="Source implementation for Confluence.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-confluence/source_confluence/run.py b/airbyte-integrations/connectors/source-confluence/source_confluence/run.py new file mode 100644 index 000000000000..b52381028de6 --- /dev/null +++ b/airbyte-integrations/connectors/source-confluence/source_confluence/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_confluence import SourceConfluence + + +def run(): + source = SourceConfluence() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-convertkit/main.py b/airbyte-integrations/connectors/source-convertkit/main.py index 95b40e4a8c42..0338150054eb 100644 --- a/airbyte-integrations/connectors/source-convertkit/main.py +++ b/airbyte-integrations/connectors/source-convertkit/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_convertkit import SourceConvertkit +from source_convertkit.run import run if __name__ == "__main__": - source = SourceConvertkit() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-convertkit/setup.py b/airbyte-integrations/connectors/source-convertkit/setup.py index a4a3f97a2b7d..1f8512be419f 100644 --- a/airbyte-integrations/connectors/source-convertkit/setup.py +++ b/airbyte-integrations/connectors/source-convertkit/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-convertkit=source_convertkit.run:run", + ], + }, name="source_convertkit", description="Source implementation for Convertkit.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-convertkit/source_convertkit/run.py b/airbyte-integrations/connectors/source-convertkit/source_convertkit/run.py new file mode 100644 index 000000000000..0db38a005bc4 --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/source_convertkit/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_convertkit import SourceConvertkit + + +def run(): + source = SourceConvertkit() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-convex/main.py b/airbyte-integrations/connectors/source-convex/main.py index 90037cfed046..751ae667fae2 100644 --- a/airbyte-integrations/connectors/source-convex/main.py +++ b/airbyte-integrations/connectors/source-convex/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_convex import SourceConvex +from source_convex.run import run if __name__ == "__main__": - source = SourceConvex() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-convex/setup.py b/airbyte-integrations/connectors/source-convex/setup.py index 22937fa853f1..82f937d289d9 100644 --- a/airbyte-integrations/connectors/source-convex/setup.py +++ b/airbyte-integrations/connectors/source-convex/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-convex=source_convex.run:run", + ], + }, name="source_convex", description="Source implementation for Convex.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-convex/source_convex/run.py b/airbyte-integrations/connectors/source-convex/source_convex/run.py new file mode 100644 index 000000000000..94ff036e088c --- /dev/null +++ b/airbyte-integrations/connectors/source-convex/source_convex/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_convex import SourceConvex + + +def run(): + source = SourceConvex() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-copper/main.py b/airbyte-integrations/connectors/source-copper/main.py index 5c45e3420a77..090988115e85 100644 --- a/airbyte-integrations/connectors/source-copper/main.py +++ b/airbyte-integrations/connectors/source-copper/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_copper import SourceCopper +from source_copper.run import run if __name__ == "__main__": - source = SourceCopper() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-copper/setup.py b/airbyte-integrations/connectors/source-copper/setup.py index 15fe92381858..f4a0f506acbb 100644 --- a/airbyte-integrations/connectors/source-copper/setup.py +++ b/airbyte-integrations/connectors/source-copper/setup.py @@ -10,13 +10,30 @@ TEST_REQUIREMENTS = ["pytest~=6.2", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-copper=source_copper.run:run", + ], + }, name="source_copper", description="Source implementation for Copper.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-copper/source_copper/run.py b/airbyte-integrations/connectors/source-copper/source_copper/run.py new file mode 100644 index 000000000000..1267cf2c4722 --- /dev/null +++ b/airbyte-integrations/connectors/source-copper/source_copper/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_copper import SourceCopper + + +def run(): + source = SourceCopper() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-courier/main.py b/airbyte-integrations/connectors/source-courier/main.py index df87edbfaf1d..d4de8fbdc917 100644 --- a/airbyte-integrations/connectors/source-courier/main.py +++ b/airbyte-integrations/connectors/source-courier/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_courier import SourceCourier +from source_courier.run import run if __name__ == "__main__": - source = SourceCourier() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-courier/setup.py b/airbyte-integrations/connectors/source-courier/setup.py index 883332ece7a6..9bb4390f1913 100644 --- a/airbyte-integrations/connectors/source-courier/setup.py +++ b/airbyte-integrations/connectors/source-courier/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-courier=source_courier.run:run", + ], + }, name="source_courier", description="Source implementation for Courier.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-courier/source_courier/run.py b/airbyte-integrations/connectors/source-courier/source_courier/run.py new file mode 100644 index 000000000000..c6e85fbde238 --- /dev/null +++ b/airbyte-integrations/connectors/source-courier/source_courier/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_courier import SourceCourier + + +def run(): + source = SourceCourier() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-customer-io/main.py b/airbyte-integrations/connectors/source-customer-io/main.py index 835bc1df92ba..ce54e7ce6db6 100644 --- a/airbyte-integrations/connectors/source-customer-io/main.py +++ b/airbyte-integrations/connectors/source-customer-io/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_customer_io import SourceCustomerIo +from source_customer_io.run import run if __name__ == "__main__": - source = SourceCustomerIo() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-customer-io/setup.py b/airbyte-integrations/connectors/source-customer-io/setup.py index 04cd8664c8a1..eb13cd73bbea 100644 --- a/airbyte-integrations/connectors/source-customer-io/setup.py +++ b/airbyte-integrations/connectors/source-customer-io/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-customer-io=source_customer_io.run:run", + ], + }, name="source_customer_io", description="Source implementation for Customer Io.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-customer-io/source_customer_io/run.py b/airbyte-integrations/connectors/source-customer-io/source_customer_io/run.py new file mode 100644 index 000000000000..b122bbd03497 --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/source_customer_io/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_customer_io import SourceCustomerIo + + +def run(): + source = SourceCustomerIo() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-datadog/main.py b/airbyte-integrations/connectors/source-datadog/main.py index 768db52beb4f..d8e7d33f093e 100644 --- a/airbyte-integrations/connectors/source-datadog/main.py +++ b/airbyte-integrations/connectors/source-datadog/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_datadog import SourceDatadog +from source_datadog.run import run if __name__ == "__main__": - source = SourceDatadog() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-datadog/setup.py b/airbyte-integrations/connectors/source-datadog/setup.py index 432515ebd8f0..59b397984d78 100644 --- a/airbyte-integrations/connectors/source-datadog/setup.py +++ b/airbyte-integrations/connectors/source-datadog/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-datadog=source_datadog.run:run", + ], + }, name="source_datadog", description="Source implementation for Datadog.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-datadog/source_datadog/run.py b/airbyte-integrations/connectors/source-datadog/source_datadog/run.py new file mode 100644 index 000000000000..62adbe197666 --- /dev/null +++ b/airbyte-integrations/connectors/source-datadog/source_datadog/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_datadog import SourceDatadog + + +def run(): + source = SourceDatadog() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-datascope/main.py b/airbyte-integrations/connectors/source-datascope/main.py index dbdd6b9cc1a5..59b428ca2396 100644 --- a/airbyte-integrations/connectors/source-datascope/main.py +++ b/airbyte-integrations/connectors/source-datascope/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_datascope import SourceDatascope +from source_datascope.run import run if __name__ == "__main__": - source = SourceDatascope() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-datascope/setup.py b/airbyte-integrations/connectors/source-datascope/setup.py index 4a3a8864ddf4..d999c1e72290 100644 --- a/airbyte-integrations/connectors/source-datascope/setup.py +++ b/airbyte-integrations/connectors/source-datascope/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-datascope=source_datascope.run:run", + ], + }, name="source_datascope", description="Source implementation for Datascope.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-datascope/source_datascope/run.py b/airbyte-integrations/connectors/source-datascope/source_datascope/run.py new file mode 100644 index 000000000000..e1543c70c39c --- /dev/null +++ b/airbyte-integrations/connectors/source-datascope/source_datascope/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_datascope import SourceDatascope + + +def run(): + source = SourceDatascope() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-delighted/main.py b/airbyte-integrations/connectors/source-delighted/main.py index 2ba66b4bf80a..f2a80745b8cd 100644 --- a/airbyte-integrations/connectors/source-delighted/main.py +++ b/airbyte-integrations/connectors/source-delighted/main.py @@ -2,34 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -import sys - -from airbyte_cdk.entrypoint import launch -from source_delighted import SourceDelighted +from source_delighted.run import run if __name__ == "__main__": - source = SourceDelighted() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-delighted/setup.py b/airbyte-integrations/connectors/source-delighted/setup.py index ca92c54ec6fe..f1f4a7be9d78 100644 --- a/airbyte-integrations/connectors/source-delighted/setup.py +++ b/airbyte-integrations/connectors/source-delighted/setup.py @@ -17,6 +17,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-delighted=source_delighted.run:run", + ], + }, name="source_delighted", description="Source implementation for Delighted.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/run.py b/airbyte-integrations/connectors/source-delighted/source_delighted/run.py new file mode 100644 index 000000000000..b38535e5962d --- /dev/null +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/run.py @@ -0,0 +1,36 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_delighted import SourceDelighted + + +def run(): + source = SourceDelighted() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-dixa/main.py b/airbyte-integrations/connectors/source-dixa/main.py index 5ec7b113ae37..cf2dc464235a 100644 --- a/airbyte-integrations/connectors/source-dixa/main.py +++ b/airbyte-integrations/connectors/source-dixa/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_dixa import SourceDixa +from source_dixa.run import run if __name__ == "__main__": - source = SourceDixa() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-dixa/setup.py b/airbyte-integrations/connectors/source-dixa/setup.py index be6537c6a3cd..5a08f6fc5552 100644 --- a/airbyte-integrations/connectors/source-dixa/setup.py +++ b/airbyte-integrations/connectors/source-dixa/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-dixa=source_dixa.run:run", + ], + }, name="source_dixa", description="Source implementation for Dixa.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-dixa/source_dixa/run.py b/airbyte-integrations/connectors/source-dixa/source_dixa/run.py new file mode 100644 index 000000000000..f264dd78af8e --- /dev/null +++ b/airbyte-integrations/connectors/source-dixa/source_dixa/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_dixa import SourceDixa + + +def run(): + source = SourceDixa() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-dockerhub/main.py b/airbyte-integrations/connectors/source-dockerhub/main.py index fffced0a26b7..c7bf0a5fadc9 100644 --- a/airbyte-integrations/connectors/source-dockerhub/main.py +++ b/airbyte-integrations/connectors/source-dockerhub/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_dockerhub import SourceDockerhub +from source_dockerhub.run import run if __name__ == "__main__": - source = SourceDockerhub() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-dockerhub/setup.py b/airbyte-integrations/connectors/source-dockerhub/setup.py index ea4b2d5873c8..9d382d831668 100644 --- a/airbyte-integrations/connectors/source-dockerhub/setup.py +++ b/airbyte-integrations/connectors/source-dockerhub/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-dockerhub=source_dockerhub.run:run", + ], + }, name="source_dockerhub", description="Source implementation for Dockerhub.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/run.py b/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/run.py new file mode 100644 index 000000000000..40df913e37e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_dockerhub import SourceDockerhub + + +def run(): + source = SourceDockerhub() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-dremio/main.py b/airbyte-integrations/connectors/source-dremio/main.py index 10c2231becd9..2a7f7fbaabd4 100644 --- a/airbyte-integrations/connectors/source-dremio/main.py +++ b/airbyte-integrations/connectors/source-dremio/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_dremio import SourceDremio +from source_dremio.run import run if __name__ == "__main__": - source = SourceDremio() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-dremio/setup.py b/airbyte-integrations/connectors/source-dremio/setup.py index 290c8fcf4fbe..0d89e8a42fe6 100644 --- a/airbyte-integrations/connectors/source-dremio/setup.py +++ b/airbyte-integrations/connectors/source-dremio/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-dremio=source_dremio.run:run", + ], + }, name="source_dremio", description="Source implementation for Dremio.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-dremio/source_dremio/run.py b/airbyte-integrations/connectors/source-dremio/source_dremio/run.py new file mode 100644 index 000000000000..8df023f7eff9 --- /dev/null +++ b/airbyte-integrations/connectors/source-dremio/source_dremio/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_dremio import SourceDremio + + +def run(): + source = SourceDremio() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-drift/main.py b/airbyte-integrations/connectors/source-drift/main.py index 0e0072ac04c3..7f9fdd2ad3df 100644 --- a/airbyte-integrations/connectors/source-drift/main.py +++ b/airbyte-integrations/connectors/source-drift/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_drift import SourceDrift +from source_drift.run import run if __name__ == "__main__": - source = SourceDrift() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-drift/setup.py b/airbyte-integrations/connectors/source-drift/setup.py index 5407ab8ae1e5..7e6632fa4725 100644 --- a/airbyte-integrations/connectors/source-drift/setup.py +++ b/airbyte-integrations/connectors/source-drift/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-drift=source_drift.run:run", + ], + }, name="source_drift", description="Source implementation for Drift.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-drift/source_drift/run.py b/airbyte-integrations/connectors/source-drift/source_drift/run.py new file mode 100644 index 000000000000..b9b04289c276 --- /dev/null +++ b/airbyte-integrations/connectors/source-drift/source_drift/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_drift import SourceDrift + + +def run(): + source = SourceDrift() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-dv-360/main.py b/airbyte-integrations/connectors/source-dv-360/main.py index 483df6f4de96..4d7158d2a11d 100644 --- a/airbyte-integrations/connectors/source-dv-360/main.py +++ b/airbyte-integrations/connectors/source-dv-360/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_dv_360 import SourceDV360 +from source_dv_360.run import run if __name__ == "__main__": - source = SourceDV360() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-dv-360/setup.py b/airbyte-integrations/connectors/source-dv-360/setup.py index a8198206559e..99188791455c 100644 --- a/airbyte-integrations/connectors/source-dv-360/setup.py +++ b/airbyte-integrations/connectors/source-dv-360/setup.py @@ -10,6 +10,11 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock"] setup( + entry_points={ + "console_scripts": [ + "source-dv-360=source_dv_360.run:run", + ], + }, name="source_dv_360", description="Source implementation for Display & Video 360.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-dv-360/source_dv_360/run.py b/airbyte-integrations/connectors/source-dv-360/source_dv_360/run.py new file mode 100644 index 000000000000..a869331c4dbf --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/source_dv_360/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_dv_360 import SourceDV360 + + +def run(): + source = SourceDV360() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-emailoctopus/main.py b/airbyte-integrations/connectors/source-emailoctopus/main.py index 18d45ef8ae6f..c0c08dfdd774 100644 --- a/airbyte-integrations/connectors/source-emailoctopus/main.py +++ b/airbyte-integrations/connectors/source-emailoctopus/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_emailoctopus import SourceEmailoctopus +from source_emailoctopus.run import run if __name__ == "__main__": - source = SourceEmailoctopus() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-emailoctopus/setup.py b/airbyte-integrations/connectors/source-emailoctopus/setup.py index 9cd42d8b7050..4c4d0b6e863c 100644 --- a/airbyte-integrations/connectors/source-emailoctopus/setup.py +++ b/airbyte-integrations/connectors/source-emailoctopus/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-emailoctopus=source_emailoctopus.run:run", + ], + }, name="source_emailoctopus", description="Source implementation for Emailoctopus.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/run.py b/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/run.py new file mode 100644 index 000000000000..407e0e086c97 --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_emailoctopus import SourceEmailoctopus + + +def run(): + source = SourceEmailoctopus() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-everhour/main.py b/airbyte-integrations/connectors/source-everhour/main.py index 8c44913d9ddf..d69ca6247ce6 100644 --- a/airbyte-integrations/connectors/source-everhour/main.py +++ b/airbyte-integrations/connectors/source-everhour/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_everhour import SourceEverhour +from source_everhour.run import run if __name__ == "__main__": - source = SourceEverhour() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-everhour/setup.py b/airbyte-integrations/connectors/source-everhour/setup.py index e1838cba9eda..40c9560ff254 100644 --- a/airbyte-integrations/connectors/source-everhour/setup.py +++ b/airbyte-integrations/connectors/source-everhour/setup.py @@ -12,13 +12,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-everhour=source_everhour.run:run", + ], + }, name="source_everhour", description="Source implementation for Everhour.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-everhour/source_everhour/run.py b/airbyte-integrations/connectors/source-everhour/source_everhour/run.py new file mode 100644 index 000000000000..4e36ccaaf781 --- /dev/null +++ b/airbyte-integrations/connectors/source-everhour/source_everhour/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_everhour import SourceEverhour + + +def run(): + source = SourceEverhour() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-exchange-rates/main.py b/airbyte-integrations/connectors/source-exchange-rates/main.py index ffdcfd706cff..ce703c9da571 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/main.py +++ b/airbyte-integrations/connectors/source-exchange-rates/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_exchange_rates import SourceExchangeRates +from source_exchange_rates.run import run if __name__ == "__main__": - source = SourceExchangeRates() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-exchange-rates/setup.py b/airbyte-integrations/connectors/source-exchange-rates/setup.py index 120437119f72..c7832abdf32d 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/setup.py +++ b/airbyte-integrations/connectors/source-exchange-rates/setup.py @@ -15,13 +15,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-exchange-rates=source_exchange_rates.run:run", + ], + }, name="source_exchange_rates", description="Source implementation for Exchange Rates.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/run.py b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/run.py new file mode 100644 index 000000000000..c710a7e23d0d --- /dev/null +++ b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_exchange_rates import SourceExchangeRates + + +def run(): + source = SourceExchangeRates() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-facebook-pages/main.py b/airbyte-integrations/connectors/source-facebook-pages/main.py index aba9ffd9f0ea..466fc2800442 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/main.py +++ b/airbyte-integrations/connectors/source-facebook-pages/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_facebook_pages import SourceFacebookPages +from source_facebook_pages.run import run if __name__ == "__main__": - source = SourceFacebookPages() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-facebook-pages/setup.py b/airbyte-integrations/connectors/source-facebook-pages/setup.py index 7bce89eaea9d..808c6ffb27c7 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/setup.py +++ b/airbyte-integrations/connectors/source-facebook-pages/setup.py @@ -12,13 +12,30 @@ TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6.1", "requests-mock"] setup( + entry_points={ + "console_scripts": [ + "source-facebook-pages=source_facebook_pages.run:run", + ], + }, name="source_facebook_pages", description="Source implementation for Facebook Pages.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/run.py b/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/run.py new file mode 100644 index 000000000000..3b70710fe59d --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_facebook_pages import SourceFacebookPages + + +def run(): + source = SourceFacebookPages() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-fastbill/main.py b/airbyte-integrations/connectors/source-fastbill/main.py index d807714bce50..acf657a8214d 100644 --- a/airbyte-integrations/connectors/source-fastbill/main.py +++ b/airbyte-integrations/connectors/source-fastbill/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_fastbill import SourceFastbill +from source_fastbill.run import run if __name__ == "__main__": - source = SourceFastbill() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-fastbill/setup.py b/airbyte-integrations/connectors/source-fastbill/setup.py index 843e992da23a..a83e4e1d307a 100644 --- a/airbyte-integrations/connectors/source-fastbill/setup.py +++ b/airbyte-integrations/connectors/source-fastbill/setup.py @@ -10,13 +10,30 @@ TEST_REQUIREMENTS = ["pytest~=6.2", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-fastbill=source_fastbill.run:run", + ], + }, name="source_fastbill", description="Source implementation for Fastbill.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-fastbill/source_fastbill/run.py b/airbyte-integrations/connectors/source-fastbill/source_fastbill/run.py new file mode 100644 index 000000000000..eee32cd7dd6c --- /dev/null +++ b/airbyte-integrations/connectors/source-fastbill/source_fastbill/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_fastbill import SourceFastbill + + +def run(): + source = SourceFastbill() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-fauna/main.py b/airbyte-integrations/connectors/source-fauna/main.py index 86b4f167af64..9e4bc25307ed 100644 --- a/airbyte-integrations/connectors/source-fauna/main.py +++ b/airbyte-integrations/connectors/source-fauna/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_fauna import SourceFauna +from source_fauna.run import run if __name__ == "__main__": - source = SourceFauna() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-fauna/setup.py b/airbyte-integrations/connectors/source-fauna/setup.py index 552523c1eb37..db1a676e8035 100644 --- a/airbyte-integrations/connectors/source-fauna/setup.py +++ b/airbyte-integrations/connectors/source-fauna/setup.py @@ -17,6 +17,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-fauna=source_fauna.run:run", + ], + }, name="source_fauna", description="Source implementation for Fauna.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-fauna/source_fauna/run.py b/airbyte-integrations/connectors/source-fauna/source_fauna/run.py new file mode 100644 index 000000000000..20d6ee8bb021 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/source_fauna/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_fauna import SourceFauna + + +def run(): + source = SourceFauna() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-file/main.py b/airbyte-integrations/connectors/source-file/main.py index 3ab698c087c0..3e7e82fd61d8 100644 --- a/airbyte-integrations/connectors/source-file/main.py +++ b/airbyte-integrations/connectors/source-file/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_file import SourceFile +from source_file.run import run if __name__ == "__main__": - source = SourceFile() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-file/setup.py b/airbyte-integrations/connectors/source-file/setup.py index 513527062877..b2ae12af1513 100644 --- a/airbyte-integrations/connectors/source-file/setup.py +++ b/airbyte-integrations/connectors/source-file/setup.py @@ -27,6 +27,11 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-docker~=2.0.1", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-file=source_file.run:run", + ], + }, name="source_file", description="Source implementation for File", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-file/source_file/run.py b/airbyte-integrations/connectors/source-file/source_file/run.py new file mode 100644 index 000000000000..646b35cb4c93 --- /dev/null +++ b/airbyte-integrations/connectors/source-file/source_file/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_file import SourceFile + + +def run(): + source = SourceFile() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/main.py b/airbyte-integrations/connectors/source-firebase-realtime-database/main.py index 54d63471838e..708648fa8c15 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/main.py +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_firebase_realtime_database import SourceFirebaseRealtimeDatabase +from source_firebase_realtime_database.run import run if __name__ == "__main__": - source = SourceFirebaseRealtimeDatabase() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/setup.py b/airbyte-integrations/connectors/source-firebase-realtime-database/setup.py index 1424bb5b1b68..84f7e112ff68 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/setup.py +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/setup.py @@ -18,6 +18,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-firebase-realtime-database=source_firebase_realtime_database.run:run", + ], + }, name="source_firebase_realtime_database", description="Source implementation for Firebase Realtime Database.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/source_firebase_realtime_database/run.py b/airbyte-integrations/connectors/source-firebase-realtime-database/source_firebase_realtime_database/run.py new file mode 100644 index 000000000000..19b835a4a7a2 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/source_firebase_realtime_database/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_firebase_realtime_database import SourceFirebaseRealtimeDatabase + + +def run(): + source = SourceFirebaseRealtimeDatabase() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-firebolt/main.py b/airbyte-integrations/connectors/source-firebolt/main.py index babb5aad001b..a901e9c4ae29 100644 --- a/airbyte-integrations/connectors/source-firebolt/main.py +++ b/airbyte-integrations/connectors/source-firebolt/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_firebolt import SourceFirebolt +from source_firebolt.run import run if __name__ == "__main__": - source = SourceFirebolt() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-firebolt/setup.py b/airbyte-integrations/connectors/source-firebolt/setup.py index 13bb7d102b18..7c342fd2ae16 100644 --- a/airbyte-integrations/connectors/source-firebolt/setup.py +++ b/airbyte-integrations/connectors/source-firebolt/setup.py @@ -15,6 +15,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-firebolt=source_firebolt.run:run", + ], + }, name="source_firebolt", description="Source implementation for Firebolt.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-firebolt/source_firebolt/run.py b/airbyte-integrations/connectors/source-firebolt/source_firebolt/run.py new file mode 100644 index 000000000000..d37e5a6ba911 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/source_firebolt/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_firebolt import SourceFirebolt + + +def run(): + source = SourceFirebolt() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-flexport/main.py b/airbyte-integrations/connectors/source-flexport/main.py index e65198a5e67d..2370e6f8868a 100644 --- a/airbyte-integrations/connectors/source-flexport/main.py +++ b/airbyte-integrations/connectors/source-flexport/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_flexport import SourceFlexport +from source_flexport.run import run if __name__ == "__main__": - source = SourceFlexport() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-flexport/setup.py b/airbyte-integrations/connectors/source-flexport/setup.py index 0deaf76a6445..8a0a2f201f9b 100644 --- a/airbyte-integrations/connectors/source-flexport/setup.py +++ b/airbyte-integrations/connectors/source-flexport/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-flexport=source_flexport.run:run", + ], + }, name="source_flexport", description="Source implementation for Flexport.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/run.py b/airbyte-integrations/connectors/source-flexport/source_flexport/run.py new file mode 100644 index 000000000000..9bba4b7e96a4 --- /dev/null +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_flexport import SourceFlexport + + +def run(): + source = SourceFlexport() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-freshcaller/main.py b/airbyte-integrations/connectors/source-freshcaller/main.py index e0bc9f142e11..7039ceb25a6d 100644 --- a/airbyte-integrations/connectors/source-freshcaller/main.py +++ b/airbyte-integrations/connectors/source-freshcaller/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_freshcaller import SourceFreshcaller +from source_freshcaller.run import run if __name__ == "__main__": - source = SourceFreshcaller() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-freshcaller/setup.py b/airbyte-integrations/connectors/source-freshcaller/setup.py index 27b47c30913f..3c1ee9c3ca91 100644 --- a/airbyte-integrations/connectors/source-freshcaller/setup.py +++ b/airbyte-integrations/connectors/source-freshcaller/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-freshcaller=source_freshcaller.run:run", + ], + }, name="source_freshcaller", description="Source implementation for Freshcaller.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/run.py b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/run.py new file mode 100644 index 000000000000..b6757d75d1aa --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_freshcaller import SourceFreshcaller + + +def run(): + source = SourceFreshcaller() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-freshsales/main.py b/airbyte-integrations/connectors/source-freshsales/main.py index c5069e9490e2..cdb22eb76f98 100644 --- a/airbyte-integrations/connectors/source-freshsales/main.py +++ b/airbyte-integrations/connectors/source-freshsales/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_freshsales import SourceFreshsales +from source_freshsales.run import run if __name__ == "__main__": - source = SourceFreshsales() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-freshsales/setup.py b/airbyte-integrations/connectors/source-freshsales/setup.py index 2cc1107f0c8b..664bc7a4ff49 100644 --- a/airbyte-integrations/connectors/source-freshsales/setup.py +++ b/airbyte-integrations/connectors/source-freshsales/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-freshsales=source_freshsales.run:run", + ], + }, name="source_freshsales", description="Source implementation for Freshsales.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/run.py b/airbyte-integrations/connectors/source-freshsales/source_freshsales/run.py new file mode 100644 index 000000000000..5eed96da068e --- /dev/null +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_freshsales import SourceFreshsales + + +def run(): + source = SourceFreshsales() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-freshservice/main.py b/airbyte-integrations/connectors/source-freshservice/main.py index 5048dde91929..084c02978df9 100644 --- a/airbyte-integrations/connectors/source-freshservice/main.py +++ b/airbyte-integrations/connectors/source-freshservice/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_freshservice import SourceFreshservice +from source_freshservice.run import run if __name__ == "__main__": - source = SourceFreshservice() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-freshservice/setup.py b/airbyte-integrations/connectors/source-freshservice/setup.py index 422531f33640..87806214b3dc 100644 --- a/airbyte-integrations/connectors/source-freshservice/setup.py +++ b/airbyte-integrations/connectors/source-freshservice/setup.py @@ -12,13 +12,30 @@ TEST_REQUIREMENTS = ["pytest~=6.2", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-freshservice=source_freshservice.run:run", + ], + }, name="source_freshservice", description="Source implementation for Freshservice.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/run.py b/airbyte-integrations/connectors/source-freshservice/source_freshservice/run.py new file mode 100644 index 000000000000..c7a979f0a59b --- /dev/null +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_freshservice import SourceFreshservice + + +def run(): + source = SourceFreshservice() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-fullstory/main.py b/airbyte-integrations/connectors/source-fullstory/main.py index d9696f00cd3d..ee3e71e54ed5 100644 --- a/airbyte-integrations/connectors/source-fullstory/main.py +++ b/airbyte-integrations/connectors/source-fullstory/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_fullstory import SourceFullstory +from source_fullstory.run import run if __name__ == "__main__": - source = SourceFullstory() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-fullstory/setup.py b/airbyte-integrations/connectors/source-fullstory/setup.py index 99bc576b124a..4bc344fd16c3 100644 --- a/airbyte-integrations/connectors/source-fullstory/setup.py +++ b/airbyte-integrations/connectors/source-fullstory/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-fullstory=source_fullstory.run:run", + ], + }, name="source_fullstory", description="Source implementation for Fullstory.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-fullstory/source_fullstory/run.py b/airbyte-integrations/connectors/source-fullstory/source_fullstory/run.py new file mode 100644 index 000000000000..3b6be606bf59 --- /dev/null +++ b/airbyte-integrations/connectors/source-fullstory/source_fullstory/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_fullstory import SourceFullstory + + +def run(): + source = SourceFullstory() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-gainsight-px/main.py b/airbyte-integrations/connectors/source-gainsight-px/main.py index 5ae4980cd0e0..35146e9ca972 100644 --- a/airbyte-integrations/connectors/source-gainsight-px/main.py +++ b/airbyte-integrations/connectors/source-gainsight-px/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_gainsight_px import SourceGainsightPx +from source_gainsight_px.run import run if __name__ == "__main__": - source = SourceGainsightPx() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-gainsight-px/setup.py b/airbyte-integrations/connectors/source-gainsight-px/setup.py index 3ba161a38e22..4510521fa81c 100644 --- a/airbyte-integrations/connectors/source-gainsight-px/setup.py +++ b/airbyte-integrations/connectors/source-gainsight-px/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-gainsight-px=source_gainsight_px.run:run", + ], + }, name="source_gainsight_px", description="Source implementation for Gainsight Px.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/run.py b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/run.py new file mode 100644 index 000000000000..f0e263867dd8 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_gainsight_px import SourceGainsightPx + + +def run(): + source = SourceGainsightPx() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-gcs/main.py b/airbyte-integrations/connectors/source-gcs/main.py index c98b5b943cc7..a3a044fb142d 100644 --- a/airbyte-integrations/connectors/source-gcs/main.py +++ b/airbyte-integrations/connectors/source-gcs/main.py @@ -2,14 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch -from source_gcs import Config, Cursor, SourceGCS, SourceGCSStreamReader +from source_gcs.run import run if __name__ == "__main__": - _args = sys.argv[1:] - catalog_path = AirbyteEntrypoint.extract_catalog(_args) - source = SourceGCS(SourceGCSStreamReader(), Config, catalog_path, cursor_cls=Cursor) - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-gcs/setup.py b/airbyte-integrations/connectors/source-gcs/setup.py index b9574a838971..f218f69dcb09 100644 --- a/airbyte-integrations/connectors/source-gcs/setup.py +++ b/airbyte-integrations/connectors/source-gcs/setup.py @@ -19,6 +19,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-gcs=source_gcs.run:run", + ], + }, name="source_gcs", description="Source implementation for Gcs.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/run.py b/airbyte-integrations/connectors/source-gcs/source_gcs/run.py new file mode 100644 index 000000000000..d91a6b40df2a --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/run.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch +from source_gcs import Config, Cursor, SourceGCS, SourceGCSStreamReader + + +def run(): + _args = sys.argv[1:] + catalog_path = AirbyteEntrypoint.extract_catalog(_args) + source = SourceGCS(SourceGCSStreamReader(), Config, catalog_path, cursor_cls=Cursor) + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-genesys/main.py b/airbyte-integrations/connectors/source-genesys/main.py index 603590f5fe12..d34643d2aa21 100644 --- a/airbyte-integrations/connectors/source-genesys/main.py +++ b/airbyte-integrations/connectors/source-genesys/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_genesys import SourceGenesys +from source_genesys.run import run if __name__ == "__main__": - source = SourceGenesys() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-genesys/setup.py b/airbyte-integrations/connectors/source-genesys/setup.py index 1c118e316680..723222c413e3 100644 --- a/airbyte-integrations/connectors/source-genesys/setup.py +++ b/airbyte-integrations/connectors/source-genesys/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-genesys=source_genesys.run:run", + ], + }, name="source_genesys", description="Source implementation for Genesys.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-genesys/source_genesys/run.py b/airbyte-integrations/connectors/source-genesys/source_genesys/run.py new file mode 100644 index 000000000000..1c7ee790a0ca --- /dev/null +++ b/airbyte-integrations/connectors/source-genesys/source_genesys/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_genesys import SourceGenesys + + +def run(): + source = SourceGenesys() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-getlago/main.py b/airbyte-integrations/connectors/source-getlago/main.py index 0748c658182d..979e8b779cc5 100644 --- a/airbyte-integrations/connectors/source-getlago/main.py +++ b/airbyte-integrations/connectors/source-getlago/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_getlago import SourceGetlago +from source_getlago.run import run if __name__ == "__main__": - source = SourceGetlago() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-getlago/setup.py b/airbyte-integrations/connectors/source-getlago/setup.py index 37d77842438d..b4d2e1a140eb 100644 --- a/airbyte-integrations/connectors/source-getlago/setup.py +++ b/airbyte-integrations/connectors/source-getlago/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-getlago=source_getlago.run:run", + ], + }, name="source_getlago", description="Source implementation for Getlago.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-getlago/source_getlago/run.py b/airbyte-integrations/connectors/source-getlago/source_getlago/run.py new file mode 100644 index 000000000000..a7822c1b112a --- /dev/null +++ b/airbyte-integrations/connectors/source-getlago/source_getlago/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_getlago import SourceGetlago + + +def run(): + source = SourceGetlago() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-github/main.py b/airbyte-integrations/connectors/source-github/main.py index aa6b652e953c..4d37ce6cccf5 100644 --- a/airbyte-integrations/connectors/source-github/main.py +++ b/airbyte-integrations/connectors/source-github/main.py @@ -2,15 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_github import SourceGithub -from source_github.config_migrations import MigrateBranch, MigrateRepository +from source_github.run import run if __name__ == "__main__": - source = SourceGithub() - MigrateRepository.migrate(sys.argv[1:], source) - MigrateBranch.migrate(sys.argv[1:], source) - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-github/setup.py b/airbyte-integrations/connectors/source-github/setup.py index 8b5f90f29e12..822fd9e37edf 100644 --- a/airbyte-integrations/connectors/source-github/setup.py +++ b/airbyte-integrations/connectors/source-github/setup.py @@ -10,6 +10,11 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.2", "responses~=0.23.1", "freezegun~=1.2"] setup( + entry_points={ + "console_scripts": [ + "source-github=source_github.run:run", + ], + }, name="source_github", description="Source implementation for Github.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-github/source_github/run.py b/airbyte-integrations/connectors/source-github/source_github/run.py new file mode 100644 index 000000000000..3abce9724842 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/run.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_github import SourceGithub +from source_github.config_migrations import MigrateBranch, MigrateRepository + + +def run(): + source = SourceGithub() + MigrateRepository.migrate(sys.argv[1:], source) + MigrateBranch.migrate(sys.argv[1:], source) + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-glassfrog/main.py b/airbyte-integrations/connectors/source-glassfrog/main.py index 22d58701db3d..f063b41ef47f 100644 --- a/airbyte-integrations/connectors/source-glassfrog/main.py +++ b/airbyte-integrations/connectors/source-glassfrog/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_glassfrog import SourceGlassfrog +from source_glassfrog.run import run if __name__ == "__main__": - source = SourceGlassfrog() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-glassfrog/setup.py b/airbyte-integrations/connectors/source-glassfrog/setup.py index e62098fca78b..531a1c2a7d87 100644 --- a/airbyte-integrations/connectors/source-glassfrog/setup.py +++ b/airbyte-integrations/connectors/source-glassfrog/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-glassfrog=source_glassfrog.run:run", + ], + }, name="source_glassfrog", description="Source implementation for Glassfrog.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/run.py b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/run.py new file mode 100644 index 000000000000..618d3873c80c --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_glassfrog import SourceGlassfrog + + +def run(): + source = SourceGlassfrog() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-gnews/main.py b/airbyte-integrations/connectors/source-gnews/main.py index 2d7fef617053..4702ac5fd364 100644 --- a/airbyte-integrations/connectors/source-gnews/main.py +++ b/airbyte-integrations/connectors/source-gnews/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_gnews import SourceGnews +from source_gnews.run import run if __name__ == "__main__": - source = SourceGnews() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-gnews/setup.py b/airbyte-integrations/connectors/source-gnews/setup.py index 3e4cf6f2cc28..564eaf258c85 100644 --- a/airbyte-integrations/connectors/source-gnews/setup.py +++ b/airbyte-integrations/connectors/source-gnews/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-gnews=source_gnews.run:run", + ], + }, name="source_gnews", description="Source implementation for Gnews.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-gnews/source_gnews/run.py b/airbyte-integrations/connectors/source-gnews/source_gnews/run.py new file mode 100644 index 000000000000..c2bf1ff536a4 --- /dev/null +++ b/airbyte-integrations/connectors/source-gnews/source_gnews/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_gnews import SourceGnews + + +def run(): + source = SourceGnews() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-gocardless/main.py b/airbyte-integrations/connectors/source-gocardless/main.py index 0c7fbe5c27ed..b7d51bd717b7 100644 --- a/airbyte-integrations/connectors/source-gocardless/main.py +++ b/airbyte-integrations/connectors/source-gocardless/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_gocardless import SourceGocardless +from source_gocardless.run import run if __name__ == "__main__": - source = SourceGocardless() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-gocardless/setup.py b/airbyte-integrations/connectors/source-gocardless/setup.py index b49d3b8111e0..46bcd590d002 100644 --- a/airbyte-integrations/connectors/source-gocardless/setup.py +++ b/airbyte-integrations/connectors/source-gocardless/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-gocardless=source_gocardless.run:run", + ], + }, name="source_gocardless", description="Source implementation for Gocardless.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-gocardless/source_gocardless/run.py b/airbyte-integrations/connectors/source-gocardless/source_gocardless/run.py new file mode 100644 index 000000000000..1884dd380ed4 --- /dev/null +++ b/airbyte-integrations/connectors/source-gocardless/source_gocardless/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_gocardless import SourceGocardless + + +def run(): + source = SourceGocardless() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-gong/main.py b/airbyte-integrations/connectors/source-gong/main.py index d30389447551..dc012c0e42c0 100644 --- a/airbyte-integrations/connectors/source-gong/main.py +++ b/airbyte-integrations/connectors/source-gong/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_gong import SourceGong +from source_gong.run import run if __name__ == "__main__": - source = SourceGong() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-gong/setup.py b/airbyte-integrations/connectors/source-gong/setup.py index e87d4ea56771..2232e3fe24a1 100644 --- a/airbyte-integrations/connectors/source-gong/setup.py +++ b/airbyte-integrations/connectors/source-gong/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-gong=source_gong.run:run", + ], + }, name="source_gong", description="Source implementation for Gong.", author="Elliot Trabac", author_email="elliot.trabac1@gmail.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-gong/source_gong/run.py b/airbyte-integrations/connectors/source-gong/source_gong/run.py new file mode 100644 index 000000000000..95da404269f9 --- /dev/null +++ b/airbyte-integrations/connectors/source-gong/source_gong/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_gong import SourceGong + + +def run(): + source = SourceGong() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/main.py b/airbyte-integrations/connectors/source-google-analytics-v4/main.py index 45b902bd6ced..3fd58bc1d5f6 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/main.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_google_analytics_v4 import SourceGoogleAnalyticsV4 +from source_google_analytics_v4.run import run if __name__ == "__main__": - source = SourceGoogleAnalyticsV4() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/setup.py b/airbyte-integrations/connectors/source-google-analytics-v4/setup.py index 62ec012a72f9..43f18fd04b37 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/setup.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/setup.py @@ -15,6 +15,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-google-analytics-v4=source_google_analytics_v4.run:run", + ], + }, name="source_google_analytics_v4", description="Source implementation for Google Analytics V4.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/run.py b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/run.py new file mode 100644 index 000000000000..ebb414319fab --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_google_analytics_v4 import SourceGoogleAnalyticsV4 + + +def run(): + source = SourceGoogleAnalyticsV4() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-directory/main.py b/airbyte-integrations/connectors/source-google-directory/main.py index 97076817e3a7..fa60e31af90e 100644 --- a/airbyte-integrations/connectors/source-google-directory/main.py +++ b/airbyte-integrations/connectors/source-google-directory/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_google_directory import SourceGoogleDirectory +from source_google_directory.run import run if __name__ == "__main__": - source = SourceGoogleDirectory() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-google-directory/setup.py b/airbyte-integrations/connectors/source-google-directory/setup.py index a4dbf5967f65..e10411bc2668 100644 --- a/airbyte-integrations/connectors/source-google-directory/setup.py +++ b/airbyte-integrations/connectors/source-google-directory/setup.py @@ -20,6 +20,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-google-directory=source_google_directory.run:run", + ], + }, name="source_google_directory", description="Source implementation for Google Directory.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-google-directory/source_google_directory/run.py b/airbyte-integrations/connectors/source-google-directory/source_google_directory/run.py new file mode 100644 index 000000000000..d7110346906b --- /dev/null +++ b/airbyte-integrations/connectors/source-google-directory/source_google_directory/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_google_directory import SourceGoogleDirectory + + +def run(): + source = SourceGoogleDirectory() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-pagespeed-insights/main.py b/airbyte-integrations/connectors/source-google-pagespeed-insights/main.py index 8265a27b1edb..956a0e47d3cb 100644 --- a/airbyte-integrations/connectors/source-google-pagespeed-insights/main.py +++ b/airbyte-integrations/connectors/source-google-pagespeed-insights/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_google_pagespeed_insights import SourceGooglePagespeedInsights +from source_google_pagespeed_insights.run import run if __name__ == "__main__": - source = SourceGooglePagespeedInsights() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-google-pagespeed-insights/setup.py b/airbyte-integrations/connectors/source-google-pagespeed-insights/setup.py index 07d5d4a738ff..e1c998d1d69d 100644 --- a/airbyte-integrations/connectors/source-google-pagespeed-insights/setup.py +++ b/airbyte-integrations/connectors/source-google-pagespeed-insights/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-google-pagespeed-insights=source_google_pagespeed_insights.run:run", + ], + }, name="source_google_pagespeed_insights", description="Source implementation for Google Pagespeed Insights.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-google-pagespeed-insights/source_google_pagespeed_insights/run.py b/airbyte-integrations/connectors/source-google-pagespeed-insights/source_google_pagespeed_insights/run.py new file mode 100644 index 000000000000..e4bb2c99def7 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-pagespeed-insights/source_google_pagespeed_insights/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_google_pagespeed_insights import SourceGooglePagespeedInsights + + +def run(): + source = SourceGooglePagespeedInsights() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-search-console/main.py b/airbyte-integrations/connectors/source-google-search-console/main.py index 117df652ca76..845383457bb7 100755 --- a/airbyte-integrations/connectors/source-google-search-console/main.py +++ b/airbyte-integrations/connectors/source-google-search-console/main.py @@ -2,16 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_google_search_console import SourceGoogleSearchConsole -from source_google_search_console.config_migrations import MigrateCustomReports +from source_google_search_console.run import run if __name__ == "__main__": - source = SourceGoogleSearchConsole() - # migrate config at runtime - MigrateCustomReports.migrate(sys.argv[1:], source) - # run the connector - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-google-search-console/setup.py b/airbyte-integrations/connectors/source-google-search-console/setup.py index fd73d6450d37..9888c3a79d5e 100755 --- a/airbyte-integrations/connectors/source-google-search-console/setup.py +++ b/airbyte-integrations/connectors/source-google-search-console/setup.py @@ -19,6 +19,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-google-search-console=source_google_search_console.run:run", + ], + }, name="source_google_search_console", description="Source implementation for Google Search Console.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/run.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/run.py new file mode 100755 index 000000000000..3de91fb3cc50 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/run.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_google_search_console import SourceGoogleSearchConsole +from source_google_search_console.config_migrations import MigrateCustomReports + + +def run(): + source = SourceGoogleSearchConsole() + # migrate config at runtime + MigrateCustomReports.migrate(sys.argv[1:], source) + # run the connector + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-sheets/setup.py b/airbyte-integrations/connectors/source-google-sheets/setup.py index 921d19138dbe..af6935a4686f 100644 --- a/airbyte-integrations/connectors/source-google-sheets/setup.py +++ b/airbyte-integrations/connectors/source-google-sheets/setup.py @@ -29,7 +29,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-google-webfonts/main.py b/airbyte-integrations/connectors/source-google-webfonts/main.py index 99b5b45a2ba9..70cd774d0a01 100644 --- a/airbyte-integrations/connectors/source-google-webfonts/main.py +++ b/airbyte-integrations/connectors/source-google-webfonts/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_google_webfonts import SourceGoogleWebfonts +from source_google_webfonts.run import run if __name__ == "__main__": - source = SourceGoogleWebfonts() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-google-webfonts/setup.py b/airbyte-integrations/connectors/source-google-webfonts/setup.py index 863003482ff1..1b50e26184ce 100644 --- a/airbyte-integrations/connectors/source-google-webfonts/setup.py +++ b/airbyte-integrations/connectors/source-google-webfonts/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-google-webfonts=source_google_webfonts.run:run", + ], + }, name="source_google_webfonts", description="Source implementation for Google Webfonts.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/run.py b/airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/run.py new file mode 100644 index 000000000000..fbce9176984d --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_google_webfonts import SourceGoogleWebfonts + + +def run(): + source = SourceGoogleWebfonts() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/main.py b/airbyte-integrations/connectors/source-google-workspace-admin-reports/main.py index 010d9559e95b..1e88c29a963d 100644 --- a/airbyte-integrations/connectors/source-google-workspace-admin-reports/main.py +++ b/airbyte-integrations/connectors/source-google-workspace-admin-reports/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_google_workspace_admin_reports import SourceGoogleWorkspaceAdminReports +from source_google_workspace_admin_reports.run import run if __name__ == "__main__": - source = SourceGoogleWorkspaceAdminReports() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/setup.py b/airbyte-integrations/connectors/source-google-workspace-admin-reports/setup.py index 7170103bf496..787e9f8981b4 100644 --- a/airbyte-integrations/connectors/source-google-workspace-admin-reports/setup.py +++ b/airbyte-integrations/connectors/source-google-workspace-admin-reports/setup.py @@ -21,6 +21,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-google-workspace-admin-reports=source_google_workspace_admin_reports.run:run", + ], + }, name="source_google_workspace_admin_reports", description="Source implementation for Google Workspace Admin Reports.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/source_google_workspace_admin_reports/run.py b/airbyte-integrations/connectors/source-google-workspace-admin-reports/source_google_workspace_admin_reports/run.py new file mode 100644 index 000000000000..b5ecb1fabd63 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-workspace-admin-reports/source_google_workspace_admin_reports/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_google_workspace_admin_reports import SourceGoogleWorkspaceAdminReports + + +def run(): + source = SourceGoogleWorkspaceAdminReports() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-greenhouse/main.py b/airbyte-integrations/connectors/source-greenhouse/main.py index 55e53e344062..e08a14b429fd 100644 --- a/airbyte-integrations/connectors/source-greenhouse/main.py +++ b/airbyte-integrations/connectors/source-greenhouse/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_greenhouse import SourceGreenhouse +from source_greenhouse.run import run if __name__ == "__main__": - source = SourceGreenhouse() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-greenhouse/setup.py b/airbyte-integrations/connectors/source-greenhouse/setup.py index 73945258c42b..3b9c02aec527 100644 --- a/airbyte-integrations/connectors/source-greenhouse/setup.py +++ b/airbyte-integrations/connectors/source-greenhouse/setup.py @@ -12,6 +12,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-greenhouse=source_greenhouse.run:run", + ], + }, name="source_greenhouse", description="Source implementation for Greenhouse.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/run.py b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/run.py new file mode 100644 index 000000000000..d82109b4b6cc --- /dev/null +++ b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_greenhouse import SourceGreenhouse + + +def run(): + source = SourceGreenhouse() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-gridly/main.py b/airbyte-integrations/connectors/source-gridly/main.py index 1e999fa0c2d0..307be6500faf 100644 --- a/airbyte-integrations/connectors/source-gridly/main.py +++ b/airbyte-integrations/connectors/source-gridly/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_gridly import SourceGridly +from source_gridly.run import run if __name__ == "__main__": - source = SourceGridly() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-gridly/setup.py b/airbyte-integrations/connectors/source-gridly/setup.py index dc08caa562c5..2b6e0bb2cc20 100644 --- a/airbyte-integrations/connectors/source-gridly/setup.py +++ b/airbyte-integrations/connectors/source-gridly/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-gridly=source_gridly.run:run", + ], + }, name="source_gridly", description="Source implementation for Gridly.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-gridly/source_gridly/run.py b/airbyte-integrations/connectors/source-gridly/source_gridly/run.py new file mode 100644 index 000000000000..2cddadbd477d --- /dev/null +++ b/airbyte-integrations/connectors/source-gridly/source_gridly/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_gridly import SourceGridly + + +def run(): + source = SourceGridly() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-gutendex/main.py b/airbyte-integrations/connectors/source-gutendex/main.py index 0fd65db37ce0..8304e254daa4 100644 --- a/airbyte-integrations/connectors/source-gutendex/main.py +++ b/airbyte-integrations/connectors/source-gutendex/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_gutendex import SourceGutendex +from source_gutendex.run import run if __name__ == "__main__": - source = SourceGutendex() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-gutendex/setup.py b/airbyte-integrations/connectors/source-gutendex/setup.py index b96103135cdf..6759689d84b5 100644 --- a/airbyte-integrations/connectors/source-gutendex/setup.py +++ b/airbyte-integrations/connectors/source-gutendex/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-gutendex=source_gutendex.run:run", + ], + }, name="source_gutendex", description="Source implementation for Gutendex.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-gutendex/source_gutendex/run.py b/airbyte-integrations/connectors/source-gutendex/source_gutendex/run.py new file mode 100644 index 000000000000..ba4bcb6755ac --- /dev/null +++ b/airbyte-integrations/connectors/source-gutendex/source_gutendex/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_gutendex import SourceGutendex + + +def run(): + source = SourceGutendex() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-harness/main.py b/airbyte-integrations/connectors/source-harness/main.py index b323465b96c8..a33c09315382 100644 --- a/airbyte-integrations/connectors/source-harness/main.py +++ b/airbyte-integrations/connectors/source-harness/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_harness import SourceHarness +from source_harness.run import run if __name__ == "__main__": - source = SourceHarness() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-harness/setup.py b/airbyte-integrations/connectors/source-harness/setup.py index 6bef3ce1447c..170c960c9500 100644 --- a/airbyte-integrations/connectors/source-harness/setup.py +++ b/airbyte-integrations/connectors/source-harness/setup.py @@ -14,13 +14,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-harness=source_harness.run:run", + ], + }, name="source_harness", description="Source implementation for Harness.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-harness/source_harness/run.py b/airbyte-integrations/connectors/source-harness/source_harness/run.py new file mode 100644 index 000000000000..544daa9407a1 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/source_harness/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_harness import SourceHarness + + +def run(): + source = SourceHarness() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-harvest/main.py b/airbyte-integrations/connectors/source-harvest/main.py index 3fa9904a1ea6..e00a49b587fd 100644 --- a/airbyte-integrations/connectors/source-harvest/main.py +++ b/airbyte-integrations/connectors/source-harvest/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_harvest import SourceHarvest +from source_harvest.run import run if __name__ == "__main__": - source = SourceHarvest() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-harvest/setup.py b/airbyte-integrations/connectors/source-harvest/setup.py index a208606a02a8..4c73f9283e52 100644 --- a/airbyte-integrations/connectors/source-harvest/setup.py +++ b/airbyte-integrations/connectors/source-harvest/setup.py @@ -16,6 +16,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-harvest=source_harvest.run:run", + ], + }, name="source_harvest", description="Source implementation for Harvest.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/run.py b/airbyte-integrations/connectors/source-harvest/source_harvest/run.py new file mode 100644 index 000000000000..53406b411f86 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_harvest import SourceHarvest + + +def run(): + source = SourceHarvest() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-hellobaton/main.py b/airbyte-integrations/connectors/source-hellobaton/main.py index 1c9ad3053e5a..17a946f7180e 100644 --- a/airbyte-integrations/connectors/source-hellobaton/main.py +++ b/airbyte-integrations/connectors/source-hellobaton/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_hellobaton import SourceHellobaton +from source_hellobaton.run import run if __name__ == "__main__": - source = SourceHellobaton() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-hellobaton/setup.py b/airbyte-integrations/connectors/source-hellobaton/setup.py index 91ef2d7cc5f0..d11d669bfcad 100644 --- a/airbyte-integrations/connectors/source-hellobaton/setup.py +++ b/airbyte-integrations/connectors/source-hellobaton/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-hellobaton=source_hellobaton.run:run", + ], + }, name="source_hellobaton", description="Source implementation for Hellobaton.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/run.py b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/run.py new file mode 100644 index 000000000000..4d005af85b84 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_hellobaton import SourceHellobaton + + +def run(): + source = SourceHellobaton() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-hubplanner/main.py b/airbyte-integrations/connectors/source-hubplanner/main.py index 23d01f5e10a6..aa973ab41009 100644 --- a/airbyte-integrations/connectors/source-hubplanner/main.py +++ b/airbyte-integrations/connectors/source-hubplanner/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_hubplanner import SourceHubplanner +from source_hubplanner.run import run if __name__ == "__main__": - source = SourceHubplanner() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-hubplanner/setup.py b/airbyte-integrations/connectors/source-hubplanner/setup.py index 5a7474d7c180..dd6d82f7fb64 100644 --- a/airbyte-integrations/connectors/source-hubplanner/setup.py +++ b/airbyte-integrations/connectors/source-hubplanner/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-hubplanner=source_hubplanner.run:run", + ], + }, name="source_hubplanner", description="Source implementation for Hubplanner.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/run.py b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/run.py new file mode 100644 index 000000000000..325f3b045b43 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_hubplanner import SourceHubplanner + + +def run(): + source = SourceHubplanner() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-hubspot/main.py b/airbyte-integrations/connectors/source-hubspot/main.py index 0f1cf13e4d10..dc073ca21ed6 100644 --- a/airbyte-integrations/connectors/source-hubspot/main.py +++ b/airbyte-integrations/connectors/source-hubspot/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_hubspot import SourceHubspot +from source_hubspot.run import run if __name__ == "__main__": - source = SourceHubspot() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-hubspot/setup.py b/airbyte-integrations/connectors/source-hubspot/setup.py index 0f0721b230ff..a1831948c262 100644 --- a/airbyte-integrations/connectors/source-hubspot/setup.py +++ b/airbyte-integrations/connectors/source-hubspot/setup.py @@ -17,6 +17,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-hubspot=source_hubspot.run:run", + ], + }, name="source_hubspot", description="Source implementation for HubSpot.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/run.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/run.py new file mode 100644 index 000000000000..26f4d0abef15 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_hubspot import SourceHubspot + + +def run(): + source = SourceHubspot() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-insightly/main.py b/airbyte-integrations/connectors/source-insightly/main.py index 5797ba54c977..a486e61080cb 100644 --- a/airbyte-integrations/connectors/source-insightly/main.py +++ b/airbyte-integrations/connectors/source-insightly/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_insightly import SourceInsightly +from source_insightly.run import run if __name__ == "__main__": - source = SourceInsightly() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-insightly/setup.py b/airbyte-integrations/connectors/source-insightly/setup.py index a3c070098791..40a7bc03b71c 100644 --- a/airbyte-integrations/connectors/source-insightly/setup.py +++ b/airbyte-integrations/connectors/source-insightly/setup.py @@ -14,13 +14,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-insightly=source_insightly.run:run", + ], + }, name="source_insightly", description="Source implementation for Insightly.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/run.py b/airbyte-integrations/connectors/source-insightly/source_insightly/run.py new file mode 100644 index 000000000000..d3b9124ae551 --- /dev/null +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_insightly import SourceInsightly + + +def run(): + source = SourceInsightly() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-instatus/main.py b/airbyte-integrations/connectors/source-instatus/main.py index dc0a480e8f7e..0e0c5af556b5 100644 --- a/airbyte-integrations/connectors/source-instatus/main.py +++ b/airbyte-integrations/connectors/source-instatus/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_instatus import SourceInstatus +from source_instatus.run import run if __name__ == "__main__": - source = SourceInstatus() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-instatus/setup.py b/airbyte-integrations/connectors/source-instatus/setup.py index 0a0586841809..8a914c480bb3 100644 --- a/airbyte-integrations/connectors/source-instatus/setup.py +++ b/airbyte-integrations/connectors/source-instatus/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-instatus=source_instatus.run:run", + ], + }, name="source_instatus", description="Source implementation for Instatus.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-instatus/source_instatus/run.py b/airbyte-integrations/connectors/source-instatus/source_instatus/run.py new file mode 100644 index 000000000000..ade50f0a9cdd --- /dev/null +++ b/airbyte-integrations/connectors/source-instatus/source_instatus/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_instatus import SourceInstatus + + +def run(): + source = SourceInstatus() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-intercom/main.py b/airbyte-integrations/connectors/source-intercom/main.py index a1f0ae6911bf..410860c90fd8 100644 --- a/airbyte-integrations/connectors/source-intercom/main.py +++ b/airbyte-integrations/connectors/source-intercom/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_intercom import SourceIntercom +from source_intercom.run import run if __name__ == "__main__": - source = SourceIntercom() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-intercom/setup.py b/airbyte-integrations/connectors/source-intercom/setup.py index 0432b7d7f10a..4aaa6c80c87f 100644 --- a/airbyte-integrations/connectors/source-intercom/setup.py +++ b/airbyte-integrations/connectors/source-intercom/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-intercom=source_intercom.run:run", + ], + }, name="source_intercom", description="Source implementation for Intercom Yaml.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/run.py b/airbyte-integrations/connectors/source-intercom/source_intercom/run.py new file mode 100644 index 000000000000..434766998b6e --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_intercom import SourceIntercom + + +def run(): + source = SourceIntercom() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-intruder/main.py b/airbyte-integrations/connectors/source-intruder/main.py index 3bb638ef0b9b..ec9ff51965a6 100644 --- a/airbyte-integrations/connectors/source-intruder/main.py +++ b/airbyte-integrations/connectors/source-intruder/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_intruder import SourceIntruder +from source_intruder.run import run if __name__ == "__main__": - source = SourceIntruder() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-intruder/setup.py b/airbyte-integrations/connectors/source-intruder/setup.py index 67cb265463db..1860d710d501 100644 --- a/airbyte-integrations/connectors/source-intruder/setup.py +++ b/airbyte-integrations/connectors/source-intruder/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-intruder=source_intruder.run:run", + ], + }, name="source_intruder", description="Source implementation for Intruder.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-intruder/source_intruder/run.py b/airbyte-integrations/connectors/source-intruder/source_intruder/run.py new file mode 100644 index 000000000000..fcdedf420bc8 --- /dev/null +++ b/airbyte-integrations/connectors/source-intruder/source_intruder/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_intruder import SourceIntruder + + +def run(): + source = SourceIntruder() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-ip2whois/main.py b/airbyte-integrations/connectors/source-ip2whois/main.py index 9ca61a68d1e1..746592c48705 100644 --- a/airbyte-integrations/connectors/source-ip2whois/main.py +++ b/airbyte-integrations/connectors/source-ip2whois/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_ip2whois import SourceIp2whois +from source_ip2whois.run import run if __name__ == "__main__": - source = SourceIp2whois() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-ip2whois/setup.py b/airbyte-integrations/connectors/source-ip2whois/setup.py index 92303a3f0ea4..10c6ca83410b 100644 --- a/airbyte-integrations/connectors/source-ip2whois/setup.py +++ b/airbyte-integrations/connectors/source-ip2whois/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-ip2whois=source_ip2whois.run:run", + ], + }, name="source_ip2whois", description="Source implementation for Ip2whois.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-ip2whois/source_ip2whois/run.py b/airbyte-integrations/connectors/source-ip2whois/source_ip2whois/run.py new file mode 100644 index 000000000000..7c3e0cda5573 --- /dev/null +++ b/airbyte-integrations/connectors/source-ip2whois/source_ip2whois/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_ip2whois import SourceIp2whois + + +def run(): + source = SourceIp2whois() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-jira/main.py b/airbyte-integrations/connectors/source-jira/main.py index 9a560c777bf7..1885b3974def 100644 --- a/airbyte-integrations/connectors/source-jira/main.py +++ b/airbyte-integrations/connectors/source-jira/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_jira import SourceJira +from source_jira.run import run if __name__ == "__main__": - source = SourceJira() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-jira/setup.py b/airbyte-integrations/connectors/source-jira/setup.py index 800525b0a829..ccce70beddfc 100644 --- a/airbyte-integrations/connectors/source-jira/setup.py +++ b/airbyte-integrations/connectors/source-jira/setup.py @@ -15,6 +15,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-jira=source_jira.run:run", + ], + }, name="source_jira", description="Source implementation for Jira.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-jira/source_jira/run.py b/airbyte-integrations/connectors/source-jira/source_jira/run.py new file mode 100644 index 000000000000..c5702721e0b8 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/source_jira/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_jira import SourceJira + + +def run(): + source = SourceJira() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-k6-cloud/main.py b/airbyte-integrations/connectors/source-k6-cloud/main.py index 5b694eda96f2..61325917a539 100644 --- a/airbyte-integrations/connectors/source-k6-cloud/main.py +++ b/airbyte-integrations/connectors/source-k6-cloud/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_k6_cloud import SourceK6Cloud +from source_k6_cloud.run import run if __name__ == "__main__": - source = SourceK6Cloud() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-k6-cloud/setup.py b/airbyte-integrations/connectors/source-k6-cloud/setup.py index 05908924fd5b..03ce14d72e5b 100644 --- a/airbyte-integrations/connectors/source-k6-cloud/setup.py +++ b/airbyte-integrations/connectors/source-k6-cloud/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-k6-cloud=source_k6_cloud.run:run", + ], + }, name="source_k6_cloud", description="Source implementation for K6 Cloud.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-k6-cloud/source_k6_cloud/run.py b/airbyte-integrations/connectors/source-k6-cloud/source_k6_cloud/run.py new file mode 100644 index 000000000000..c5103e21aa21 --- /dev/null +++ b/airbyte-integrations/connectors/source-k6-cloud/source_k6_cloud/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_k6_cloud import SourceK6Cloud + + +def run(): + source = SourceK6Cloud() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-klarna/main.py b/airbyte-integrations/connectors/source-klarna/main.py index 4abe77680d8b..566bf5c666d6 100644 --- a/airbyte-integrations/connectors/source-klarna/main.py +++ b/airbyte-integrations/connectors/source-klarna/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_klarna import SourceKlarna +from source_klarna.run import run if __name__ == "__main__": - source = SourceKlarna() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-klarna/setup.py b/airbyte-integrations/connectors/source-klarna/setup.py index a4742e88dd56..58609f375fe3 100644 --- a/airbyte-integrations/connectors/source-klarna/setup.py +++ b/airbyte-integrations/connectors/source-klarna/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-klarna=source_klarna.run:run", + ], + }, name="source_klarna", description="Source implementation for Klarna.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-klarna/source_klarna/run.py b/airbyte-integrations/connectors/source-klarna/source_klarna/run.py new file mode 100644 index 000000000000..8ae916c3c4e7 --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/source_klarna/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_klarna import SourceKlarna + + +def run(): + source = SourceKlarna() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-klaus-api/main.py b/airbyte-integrations/connectors/source-klaus-api/main.py index 9be6460ab485..7896d99aec94 100644 --- a/airbyte-integrations/connectors/source-klaus-api/main.py +++ b/airbyte-integrations/connectors/source-klaus-api/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_klaus_api import SourceKlausApi +from source_klaus_api.run import run if __name__ == "__main__": - source = SourceKlausApi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-klaus-api/setup.py b/airbyte-integrations/connectors/source-klaus-api/setup.py index b4444f99dd06..815b95129848 100644 --- a/airbyte-integrations/connectors/source-klaus-api/setup.py +++ b/airbyte-integrations/connectors/source-klaus-api/setup.py @@ -15,13 +15,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-klaus-api=source_klaus_api.run:run", + ], + }, name="source_klaus_api", description="Source implementation for Klaus Api.", author="Deke Li", author_email="deke.li@sendinblue.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/run.py b/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/run.py new file mode 100644 index 000000000000..c000d3864646 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_klaus_api import SourceKlausApi + + +def run(): + source = SourceKlausApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-klaviyo/main.py b/airbyte-integrations/connectors/source-klaviyo/main.py index 6285473bf2cd..5b8c871c3d7d 100644 --- a/airbyte-integrations/connectors/source-klaviyo/main.py +++ b/airbyte-integrations/connectors/source-klaviyo/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_klaviyo import SourceKlaviyo +from source_klaviyo.run import run if __name__ == "__main__": - source = SourceKlaviyo() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-klaviyo/setup.py b/airbyte-integrations/connectors/source-klaviyo/setup.py index 32a31edb0848..d9ac3a2fb807 100644 --- a/airbyte-integrations/connectors/source-klaviyo/setup.py +++ b/airbyte-integrations/connectors/source-klaviyo/setup.py @@ -10,6 +10,11 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock", "requests_mock~=1.8"] setup( + entry_points={ + "console_scripts": [ + "source-klaviyo=source_klaviyo.run:run", + ], + }, name="source_klaviyo", description="Source implementation for Klaviyo.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/run.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/run.py new file mode 100644 index 000000000000..afcae2272e29 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_klaviyo import SourceKlaviyo + + +def run(): + source = SourceKlaviyo() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-kustomer-singer/main.py b/airbyte-integrations/connectors/source-kustomer-singer/main.py index 77ee51ee831c..00920dc58a36 100644 --- a/airbyte-integrations/connectors/source-kustomer-singer/main.py +++ b/airbyte-integrations/connectors/source-kustomer-singer/main.py @@ -2,11 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import sys - -from airbyte_cdk.entrypoint import launch -from source_kustomer_singer import SourceKustomerSinger +from source_kustomer_singer.run import run if __name__ == "__main__": - source = SourceKustomerSinger() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-kustomer-singer/setup.py b/airbyte-integrations/connectors/source-kustomer-singer/setup.py index 5e14ee6e2e1d..dbaf47851ce7 100644 --- a/airbyte-integrations/connectors/source-kustomer-singer/setup.py +++ b/airbyte-integrations/connectors/source-kustomer-singer/setup.py @@ -53,6 +53,11 @@ def run(self): TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1"] setup( + entry_points={ + "console_scripts": [ + "source-kustomer-singer=source_kustomer_singer.run:run", + ], + }, name="source_kustomer_singer", description="Source implementation for Kustomer, built on the Singer tap implementation.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-kustomer-singer/source_kustomer_singer/run.py b/airbyte-integrations/connectors/source-kustomer-singer/source_kustomer_singer/run.py new file mode 100644 index 000000000000..fba603c749e8 --- /dev/null +++ b/airbyte-integrations/connectors/source-kustomer-singer/source_kustomer_singer/run.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import sys + +from airbyte_cdk.entrypoint import launch +from source_kustomer_singer import SourceKustomerSinger + + +def run(): + source = SourceKustomerSinger() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-kyriba/main.py b/airbyte-integrations/connectors/source-kyriba/main.py index 771b738d1b1b..cd0b8f1f2f3e 100644 --- a/airbyte-integrations/connectors/source-kyriba/main.py +++ b/airbyte-integrations/connectors/source-kyriba/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_kyriba import SourceKyriba +from source_kyriba.run import run if __name__ == "__main__": - source = SourceKyriba() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-kyriba/setup.py b/airbyte-integrations/connectors/source-kyriba/setup.py index c81f4d4e1e0d..00c6c13f5657 100644 --- a/airbyte-integrations/connectors/source-kyriba/setup.py +++ b/airbyte-integrations/connectors/source-kyriba/setup.py @@ -16,6 +16,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-kyriba=source_kyriba.run:run", + ], + }, name="source_kyriba", description="Source implementation for Kyriba.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-kyriba/source_kyriba/run.py b/airbyte-integrations/connectors/source-kyriba/source_kyriba/run.py new file mode 100644 index 000000000000..e12facaaab47 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyriba/source_kyriba/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_kyriba import SourceKyriba + + +def run(): + source = SourceKyriba() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-kyve/main.py b/airbyte-integrations/connectors/source-kyve/main.py index f4055d71e80e..a3740b34d958 100644 --- a/airbyte-integrations/connectors/source-kyve/main.py +++ b/airbyte-integrations/connectors/source-kyve/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_kyve import SourceKyve +from source_kyve.run import run if __name__ == "__main__": - source = SourceKyve() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-kyve/setup.py b/airbyte-integrations/connectors/source-kyve/setup.py index 1d6d5bbde662..128e97f87d25 100644 --- a/airbyte-integrations/connectors/source-kyve/setup.py +++ b/airbyte-integrations/connectors/source-kyve/setup.py @@ -16,6 +16,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-kyve=source_kyve.run:run", + ], + }, name="source_kyve", description="Source implementation for KYVE.", author="KYVE Core Team", diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/run.py b/airbyte-integrations/connectors/source-kyve/source_kyve/run.py new file mode 100644 index 000000000000..fc9753ebf2bb --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_kyve import SourceKyve + + +def run(): + source = SourceKyve() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-launchdarkly/main.py b/airbyte-integrations/connectors/source-launchdarkly/main.py index beb7cab5ae2f..997cfda7e441 100644 --- a/airbyte-integrations/connectors/source-launchdarkly/main.py +++ b/airbyte-integrations/connectors/source-launchdarkly/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_launchdarkly import SourceLaunchdarkly +from source_launchdarkly.run import run if __name__ == "__main__": - source = SourceLaunchdarkly() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-launchdarkly/setup.py b/airbyte-integrations/connectors/source-launchdarkly/setup.py index 07d48e22e847..722dfc7a0768 100644 --- a/airbyte-integrations/connectors/source-launchdarkly/setup.py +++ b/airbyte-integrations/connectors/source-launchdarkly/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-launchdarkly=source_launchdarkly.run:run", + ], + }, name="source_launchdarkly", description="Source implementation for Launchdarkly.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-launchdarkly/source_launchdarkly/run.py b/airbyte-integrations/connectors/source-launchdarkly/source_launchdarkly/run.py new file mode 100644 index 000000000000..9807f36d4a6e --- /dev/null +++ b/airbyte-integrations/connectors/source-launchdarkly/source_launchdarkly/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_launchdarkly import SourceLaunchdarkly + + +def run(): + source = SourceLaunchdarkly() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-lemlist/main.py b/airbyte-integrations/connectors/source-lemlist/main.py index 3d71a2f889d8..5d7573f45162 100644 --- a/airbyte-integrations/connectors/source-lemlist/main.py +++ b/airbyte-integrations/connectors/source-lemlist/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_lemlist import SourceLemlist +from source_lemlist.run import run if __name__ == "__main__": - source = SourceLemlist() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-lemlist/setup.py b/airbyte-integrations/connectors/source-lemlist/setup.py index b202001b47e2..53550286882a 100644 --- a/airbyte-integrations/connectors/source-lemlist/setup.py +++ b/airbyte-integrations/connectors/source-lemlist/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-lemlist=source_lemlist.run:run", + ], + }, name="source_lemlist", description="Source implementation for Lemlist.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/run.py b/airbyte-integrations/connectors/source-lemlist/source_lemlist/run.py new file mode 100644 index 000000000000..092c7e718f2b --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/source_lemlist/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_lemlist import SourceLemlist + + +def run(): + source = SourceLemlist() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-lever-hiring/main.py b/airbyte-integrations/connectors/source-lever-hiring/main.py index b79de94f75ee..ca4773e96a03 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/main.py +++ b/airbyte-integrations/connectors/source-lever-hiring/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_lever_hiring import SourceLeverHiring +from source_lever_hiring.run import run if __name__ == "__main__": - source = SourceLeverHiring() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-lever-hiring/setup.py b/airbyte-integrations/connectors/source-lever-hiring/setup.py index 3ceb890c8e93..0004a96f9d05 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/setup.py +++ b/airbyte-integrations/connectors/source-lever-hiring/setup.py @@ -17,6 +17,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-lever-hiring=source_lever_hiring.run:run", + ], + }, name="source_lever_hiring", description="Source implementation for Lever Hiring.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/run.py b/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/run.py new file mode 100644 index 000000000000..cd8dc4e7bfc3 --- /dev/null +++ b/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_lever_hiring import SourceLeverHiring + + +def run(): + source = SourceLeverHiring() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-linkedin-pages/main.py b/airbyte-integrations/connectors/source-linkedin-pages/main.py index 74ee9b7342f2..a6068720a6a8 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/main.py +++ b/airbyte-integrations/connectors/source-linkedin-pages/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_linkedin_pages import SourceLinkedinPages +from source_linkedin_pages.run import run if __name__ == "__main__": - source = SourceLinkedinPages() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-linkedin-pages/setup.py b/airbyte-integrations/connectors/source-linkedin-pages/setup.py index f690842da795..6f3d0697a13d 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/setup.py +++ b/airbyte-integrations/connectors/source-linkedin-pages/setup.py @@ -17,6 +17,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-linkedin-pages=source_linkedin_pages.run:run", + ], + }, name="source_linkedin_pages", description="Source implementation for Linkedin Company Pages.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/run.py b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/run.py new file mode 100644 index 000000000000..4f0787573f2e --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_linkedin_pages import SourceLinkedinPages + + +def run(): + source = SourceLinkedinPages() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-linnworks/main.py b/airbyte-integrations/connectors/source-linnworks/main.py index 7d0337b4c1b1..ee964c061ce0 100644 --- a/airbyte-integrations/connectors/source-linnworks/main.py +++ b/airbyte-integrations/connectors/source-linnworks/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_linnworks import SourceLinnworks +from source_linnworks.run import run if __name__ == "__main__": - source = SourceLinnworks() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-linnworks/setup.py b/airbyte-integrations/connectors/source-linnworks/setup.py index f0a487191567..1256710cb3ff 100644 --- a/airbyte-integrations/connectors/source-linnworks/setup.py +++ b/airbyte-integrations/connectors/source-linnworks/setup.py @@ -16,6 +16,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-linnworks=source_linnworks.run:run", + ], + }, name="source_linnworks", description="Source implementation for Linnworks.", author="Labanoras Tech", diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/run.py b/airbyte-integrations/connectors/source-linnworks/source_linnworks/run.py new file mode 100644 index 000000000000..d08e84e3ae6c --- /dev/null +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_linnworks import SourceLinnworks + + +def run(): + source = SourceLinnworks() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-lokalise/main.py b/airbyte-integrations/connectors/source-lokalise/main.py index 4d4423bbe36a..d02dfd231989 100644 --- a/airbyte-integrations/connectors/source-lokalise/main.py +++ b/airbyte-integrations/connectors/source-lokalise/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_lokalise import SourceLokalise +from source_lokalise.run import run if __name__ == "__main__": - source = SourceLokalise() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-lokalise/setup.py b/airbyte-integrations/connectors/source-lokalise/setup.py index 69d13c0b79f0..b2a8b5b467ec 100644 --- a/airbyte-integrations/connectors/source-lokalise/setup.py +++ b/airbyte-integrations/connectors/source-lokalise/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-lokalise=source_lokalise.run:run", + ], + }, name="source_lokalise", description="Source implementation for Lokalise.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-lokalise/source_lokalise/run.py b/airbyte-integrations/connectors/source-lokalise/source_lokalise/run.py new file mode 100644 index 000000000000..21002eb93fee --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/source_lokalise/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_lokalise import SourceLokalise + + +def run(): + source = SourceLokalise() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-looker/main.py b/airbyte-integrations/connectors/source-looker/main.py index dee0fd4beb4e..b6164cb0322b 100644 --- a/airbyte-integrations/connectors/source-looker/main.py +++ b/airbyte-integrations/connectors/source-looker/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_looker import SourceLooker +from source_looker.run import run if __name__ == "__main__": - source = SourceLooker() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-looker/setup.py b/airbyte-integrations/connectors/source-looker/setup.py index f38fc2a1095e..dc5e3295ad40 100644 --- a/airbyte-integrations/connectors/source-looker/setup.py +++ b/airbyte-integrations/connectors/source-looker/setup.py @@ -20,6 +20,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-looker=source_looker.run:run", + ], + }, name="source_looker", description="Source implementation for Looker.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-looker/source_looker/run.py b/airbyte-integrations/connectors/source-looker/source_looker/run.py new file mode 100644 index 000000000000..53aaad83ea7f --- /dev/null +++ b/airbyte-integrations/connectors/source-looker/source_looker/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_looker import SourceLooker + + +def run(): + source = SourceLooker() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-mailerlite/main.py b/airbyte-integrations/connectors/source-mailerlite/main.py index 9c906a15e012..1f2347a95e63 100644 --- a/airbyte-integrations/connectors/source-mailerlite/main.py +++ b/airbyte-integrations/connectors/source-mailerlite/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_mailerlite import SourceMailerlite +from source_mailerlite.run import run if __name__ == "__main__": - source = SourceMailerlite() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-mailerlite/setup.py b/airbyte-integrations/connectors/source-mailerlite/setup.py index 5b49cb1a5d31..71fc89f1f56a 100644 --- a/airbyte-integrations/connectors/source-mailerlite/setup.py +++ b/airbyte-integrations/connectors/source-mailerlite/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-mailerlite=source_mailerlite.run:run", + ], + }, name="source_mailerlite", description="Source implementation for Mailerlite.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/run.py b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/run.py new file mode 100644 index 000000000000..c1c7548459e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_mailerlite import SourceMailerlite + + +def run(): + source = SourceMailerlite() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-mailersend/main.py b/airbyte-integrations/connectors/source-mailersend/main.py index 8b72a409be46..9afae9104c15 100644 --- a/airbyte-integrations/connectors/source-mailersend/main.py +++ b/airbyte-integrations/connectors/source-mailersend/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_mailersend import SourceMailersend +from source_mailersend.run import run if __name__ == "__main__": - source = SourceMailersend() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-mailersend/setup.py b/airbyte-integrations/connectors/source-mailersend/setup.py index 8fc9c7d2bdbc..33d180e17b8f 100644 --- a/airbyte-integrations/connectors/source-mailersend/setup.py +++ b/airbyte-integrations/connectors/source-mailersend/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-mailersend=source_mailersend.run:run", + ], + }, name="source_mailersend", description="Source implementation for Mailersend.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-mailersend/source_mailersend/run.py b/airbyte-integrations/connectors/source-mailersend/source_mailersend/run.py new file mode 100644 index 000000000000..ddc9feff24b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/source_mailersend/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_mailersend import SourceMailersend + + +def run(): + source = SourceMailersend() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-mailgun/main.py b/airbyte-integrations/connectors/source-mailgun/main.py index bffb131e2745..ee1ad2379837 100644 --- a/airbyte-integrations/connectors/source-mailgun/main.py +++ b/airbyte-integrations/connectors/source-mailgun/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_mailgun import SourceMailgun +from source_mailgun.run import run if __name__ == "__main__": - source = SourceMailgun() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-mailgun/setup.py b/airbyte-integrations/connectors/source-mailgun/setup.py index f7661245f2fd..616448f500d5 100644 --- a/airbyte-integrations/connectors/source-mailgun/setup.py +++ b/airbyte-integrations/connectors/source-mailgun/setup.py @@ -15,13 +15,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-mailgun=source_mailgun.run:run", + ], + }, name="source_mailgun", description="Source implementation for Mailgun.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-mailgun/source_mailgun/run.py b/airbyte-integrations/connectors/source-mailgun/source_mailgun/run.py new file mode 100644 index 000000000000..6cda7fec6e71 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailgun/source_mailgun/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_mailgun import SourceMailgun + + +def run(): + source = SourceMailgun() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-mailjet-mail/main.py b/airbyte-integrations/connectors/source-mailjet-mail/main.py index 46dc79c4b5f7..5d8c4b9df24e 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/main.py +++ b/airbyte-integrations/connectors/source-mailjet-mail/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_mailjet_mail import SourceMailjetMail +from source_mailjet_mail.run import run if __name__ == "__main__": - source = SourceMailjetMail() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-mailjet-mail/setup.py b/airbyte-integrations/connectors/source-mailjet-mail/setup.py index 5f6daf3cdc2b..ea60824d3a2c 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/setup.py +++ b/airbyte-integrations/connectors/source-mailjet-mail/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-mailjet-mail=source_mailjet_mail.run:run", + ], + }, name="source_mailjet_mail", description="Source implementation for Mailjet Mail.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/run.py b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/run.py new file mode 100644 index 000000000000..87dcd7ecc198 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_mailjet_mail import SourceMailjetMail + + +def run(): + source = SourceMailjetMail() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-mailjet-sms/main.py b/airbyte-integrations/connectors/source-mailjet-sms/main.py index 7dfd0868672e..52420c275761 100644 --- a/airbyte-integrations/connectors/source-mailjet-sms/main.py +++ b/airbyte-integrations/connectors/source-mailjet-sms/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_mailjet_sms import SourceMailjetSms +from source_mailjet_sms.run import run if __name__ == "__main__": - source = SourceMailjetSms() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-mailjet-sms/setup.py b/airbyte-integrations/connectors/source-mailjet-sms/setup.py index 7edfd5e35e8e..6b4be21374cc 100644 --- a/airbyte-integrations/connectors/source-mailjet-sms/setup.py +++ b/airbyte-integrations/connectors/source-mailjet-sms/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-mailjet-sms=source_mailjet_sms.run:run", + ], + }, name="source_mailjet_sms", description="Source implementation for Mailjet Sms.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/run.py b/airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/run.py new file mode 100644 index 000000000000..a8568fdb2f68 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_mailjet_sms import SourceMailjetSms + + +def run(): + source = SourceMailjetSms() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-merge/main.py b/airbyte-integrations/connectors/source-merge/main.py index 53420fa4b3c1..694c83302b0b 100644 --- a/airbyte-integrations/connectors/source-merge/main.py +++ b/airbyte-integrations/connectors/source-merge/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_merge import SourceMerge +from source_merge.run import run if __name__ == "__main__": - source = SourceMerge() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-merge/setup.py b/airbyte-integrations/connectors/source-merge/setup.py index aa45d3bdc5e2..b403aa227f4c 100644 --- a/airbyte-integrations/connectors/source-merge/setup.py +++ b/airbyte-integrations/connectors/source-merge/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-merge=source_merge.run:run", + ], + }, name="source_merge", description="Source implementation for Merge.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-merge/source_merge/run.py b/airbyte-integrations/connectors/source-merge/source_merge/run.py new file mode 100644 index 000000000000..63187bee4753 --- /dev/null +++ b/airbyte-integrations/connectors/source-merge/source_merge/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_merge import SourceMerge + + +def run(): + source = SourceMerge() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-metabase/main.py b/airbyte-integrations/connectors/source-metabase/main.py index aeaaab111542..1e571d2ddb48 100644 --- a/airbyte-integrations/connectors/source-metabase/main.py +++ b/airbyte-integrations/connectors/source-metabase/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_metabase import SourceMetabase +from source_metabase.run import run if __name__ == "__main__": - source = SourceMetabase() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-metabase/setup.py b/airbyte-integrations/connectors/source-metabase/setup.py index 228d443e50e2..e6772737da24 100644 --- a/airbyte-integrations/connectors/source-metabase/setup.py +++ b/airbyte-integrations/connectors/source-metabase/setup.py @@ -15,13 +15,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-metabase=source_metabase.run:run", + ], + }, name="source_metabase", description="Source implementation for Metabase.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-metabase/source_metabase/run.py b/airbyte-integrations/connectors/source-metabase/source_metabase/run.py new file mode 100644 index 000000000000..90df278478af --- /dev/null +++ b/airbyte-integrations/connectors/source-metabase/source_metabase/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_metabase import SourceMetabase + + +def run(): + source = SourceMetabase() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/main.py b/airbyte-integrations/connectors/source-microsoft-dataverse/main.py index 7cf530c14f54..88b4cf3808e8 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/main.py +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_microsoft_dataverse import SourceMicrosoftDataverse +from source_microsoft_dataverse.run import run if __name__ == "__main__": - source = SourceMicrosoftDataverse() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/setup.py b/airbyte-integrations/connectors/source-microsoft-dataverse/setup.py index e86306c30138..57920ae5cd4a 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/setup.py +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-microsoft-dataverse=source_microsoft_dataverse.run:run", + ], + }, name="source_microsoft_dataverse", description="Source implementation for Microsoft Dataverse.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/run.py b/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/run.py new file mode 100644 index 000000000000..b5e6982f7292 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_microsoft_dataverse import SourceMicrosoftDataverse + + +def run(): + source = SourceMicrosoftDataverse() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/main.py b/airbyte-integrations/connectors/source-microsoft-onedrive/main.py index 205effefe600..c9280337d108 100644 --- a/airbyte-integrations/connectors/source-microsoft-onedrive/main.py +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/main.py @@ -2,15 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk import AirbyteEntrypoint -from airbyte_cdk.entrypoint import launch -from source_microsoft_onedrive import SourceMicrosoftOneDrive +from source_microsoft_onedrive.run import run if __name__ == "__main__": - args = sys.argv[1:] - catalog_path = AirbyteEntrypoint.extract_catalog(args) - source = SourceMicrosoftOneDrive(catalog_path) - launch(source, args) + run() diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/setup.py b/airbyte-integrations/connectors/source-microsoft-onedrive/setup.py index c8ed1601ac4e..8709208c5be2 100644 --- a/airbyte-integrations/connectors/source-microsoft-onedrive/setup.py +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/setup.py @@ -19,13 +19,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-microsoft-onedrive=source_microsoft_onedrive.run:run", + ], + }, name="source_microsoft_onedrive", description="Source implementation for Microsoft OneDrive.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/run.py b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/run.py new file mode 100644 index 000000000000..2141089780b9 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/run.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk import AirbyteEntrypoint +from airbyte_cdk.entrypoint import launch +from source_microsoft_onedrive import SourceMicrosoftOneDrive + + +def run(): + args = sys.argv[1:] + catalog_path = AirbyteEntrypoint.extract_catalog(args) + source = SourceMicrosoftOneDrive(catalog_path) + launch(source, args) diff --git a/airbyte-integrations/connectors/source-microsoft-teams/main.py b/airbyte-integrations/connectors/source-microsoft-teams/main.py index c8bc03c5deea..213b13e85ee5 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/main.py +++ b/airbyte-integrations/connectors/source-microsoft-teams/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_microsoft_teams import SourceMicrosoftTeams +from source_microsoft_teams.run import run if __name__ == "__main__": - source = SourceMicrosoftTeams() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-microsoft-teams/setup.py b/airbyte-integrations/connectors/source-microsoft-teams/setup.py index 6cc04d3f3b07..f840cc27859a 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/setup.py +++ b/airbyte-integrations/connectors/source-microsoft-teams/setup.py @@ -19,6 +19,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-microsoft-teams=source_microsoft_teams.run:run", + ], + }, name="source_microsoft_teams", description="Source implementation for Microsoft Teams.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/run.py b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/run.py new file mode 100644 index 000000000000..f22e12386ac7 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_microsoft_teams import SourceMicrosoftTeams + + +def run(): + source = SourceMicrosoftTeams() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-monday/main.py b/airbyte-integrations/connectors/source-monday/main.py index 6fec8ed55e58..14f4fa2d0439 100644 --- a/airbyte-integrations/connectors/source-monday/main.py +++ b/airbyte-integrations/connectors/source-monday/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_monday import SourceMonday +from source_monday.run import run if __name__ == "__main__": - source = SourceMonday() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-monday/setup.py b/airbyte-integrations/connectors/source-monday/setup.py index 4af4bd4dac1c..c99ddb8f0ed0 100644 --- a/airbyte-integrations/connectors/source-monday/setup.py +++ b/airbyte-integrations/connectors/source-monday/setup.py @@ -16,6 +16,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-monday=source_monday.run:run", + ], + }, name="source_monday", description="Source implementation for Monday.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-monday/source_monday/run.py b/airbyte-integrations/connectors/source-monday/source_monday/run.py new file mode 100644 index 000000000000..9ef2d048d213 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/source_monday/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_monday import SourceMonday + + +def run(): + source = SourceMonday() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-my-hours/main.py b/airbyte-integrations/connectors/source-my-hours/main.py index 050526fca166..d4e838cb4810 100644 --- a/airbyte-integrations/connectors/source-my-hours/main.py +++ b/airbyte-integrations/connectors/source-my-hours/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_my_hours import SourceMyHours +from source_my_hours.run import run if __name__ == "__main__": - source = SourceMyHours() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-my-hours/setup.py b/airbyte-integrations/connectors/source-my-hours/setup.py index eb4d9a7c9734..4e8a87ce39ed 100644 --- a/airbyte-integrations/connectors/source-my-hours/setup.py +++ b/airbyte-integrations/connectors/source-my-hours/setup.py @@ -17,6 +17,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-my-hours=source_my_hours.run:run", + ], + }, name="source_my_hours", description="Source implementation for My Hours.", author="Wisse Jelgersma", diff --git a/airbyte-integrations/connectors/source-my-hours/source_my_hours/run.py b/airbyte-integrations/connectors/source-my-hours/source_my_hours/run.py new file mode 100644 index 000000000000..adad265e0bac --- /dev/null +++ b/airbyte-integrations/connectors/source-my-hours/source_my_hours/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_my_hours import SourceMyHours + + +def run(): + source = SourceMyHours() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-n8n/main.py b/airbyte-integrations/connectors/source-n8n/main.py index 0762610cb5f1..b40f7c348752 100644 --- a/airbyte-integrations/connectors/source-n8n/main.py +++ b/airbyte-integrations/connectors/source-n8n/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_n8n import SourceN8n +from source_n8n.run import run if __name__ == "__main__": - source = SourceN8n() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-n8n/setup.py b/airbyte-integrations/connectors/source-n8n/setup.py index 82cdff7f8589..3592a5b8939f 100644 --- a/airbyte-integrations/connectors/source-n8n/setup.py +++ b/airbyte-integrations/connectors/source-n8n/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-n8n=source_n8n.run:run", + ], + }, name="source_n8n", description="Source implementation for N8n.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-n8n/source_n8n/run.py b/airbyte-integrations/connectors/source-n8n/source_n8n/run.py new file mode 100644 index 000000000000..04f48f6c0696 --- /dev/null +++ b/airbyte-integrations/connectors/source-n8n/source_n8n/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_n8n import SourceN8n + + +def run(): + source = SourceN8n() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-nasa/main.py b/airbyte-integrations/connectors/source-nasa/main.py index c1627273e227..5887db860888 100644 --- a/airbyte-integrations/connectors/source-nasa/main.py +++ b/airbyte-integrations/connectors/source-nasa/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_nasa import SourceNasa +from source_nasa.run import run if __name__ == "__main__": - source = SourceNasa() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-nasa/setup.py b/airbyte-integrations/connectors/source-nasa/setup.py index ecd81e7b2fd4..849b81fce45d 100644 --- a/airbyte-integrations/connectors/source-nasa/setup.py +++ b/airbyte-integrations/connectors/source-nasa/setup.py @@ -10,13 +10,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-nasa=source_nasa.run:run", + ], + }, name="source_nasa", description="Source implementation for Nasa.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-nasa/source_nasa/run.py b/airbyte-integrations/connectors/source-nasa/source_nasa/run.py new file mode 100644 index 000000000000..6b14f0280427 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/source_nasa/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_nasa import SourceNasa + + +def run(): + source = SourceNasa() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-netsuite/main.py b/airbyte-integrations/connectors/source-netsuite/main.py index 7b88a055cfe2..492266da15e2 100644 --- a/airbyte-integrations/connectors/source-netsuite/main.py +++ b/airbyte-integrations/connectors/source-netsuite/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_netsuite import SourceNetsuite +from source_netsuite.run import run if __name__ == "__main__": - source = SourceNetsuite() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-netsuite/setup.py b/airbyte-integrations/connectors/source-netsuite/setup.py index 42288908347c..e16d4d5b270f 100644 --- a/airbyte-integrations/connectors/source-netsuite/setup.py +++ b/airbyte-integrations/connectors/source-netsuite/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-netsuite=source_netsuite.run:run", + ], + }, name="source_netsuite", description="Source implementation for Netsuite Soap.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-netsuite/source_netsuite/run.py b/airbyte-integrations/connectors/source-netsuite/source_netsuite/run.py new file mode 100644 index 000000000000..c1a2043c455d --- /dev/null +++ b/airbyte-integrations/connectors/source-netsuite/source_netsuite/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_netsuite import SourceNetsuite + + +def run(): + source = SourceNetsuite() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-news-api/main.py b/airbyte-integrations/connectors/source-news-api/main.py index 0b2c02a55247..835032115f02 100644 --- a/airbyte-integrations/connectors/source-news-api/main.py +++ b/airbyte-integrations/connectors/source-news-api/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_news_api import SourceNewsApi +from source_news_api.run import run if __name__ == "__main__": - source = SourceNewsApi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-news-api/setup.py b/airbyte-integrations/connectors/source-news-api/setup.py index 733fc131e9ab..db15b71d90fa 100644 --- a/airbyte-integrations/connectors/source-news-api/setup.py +++ b/airbyte-integrations/connectors/source-news-api/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-news-api=source_news_api.run:run", + ], + }, name="source_news_api", description="Source implementation for News Api.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-news-api/source_news_api/run.py b/airbyte-integrations/connectors/source-news-api/source_news_api/run.py new file mode 100644 index 000000000000..188ce3f53c6f --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/source_news_api/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_news_api import SourceNewsApi + + +def run(): + source = SourceNewsApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-newsdata/main.py b/airbyte-integrations/connectors/source-newsdata/main.py index 2f7c8b1ecbac..81dc024c6fb2 100644 --- a/airbyte-integrations/connectors/source-newsdata/main.py +++ b/airbyte-integrations/connectors/source-newsdata/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_newsdata import SourceNewsdata +from source_newsdata.run import run if __name__ == "__main__": - source = SourceNewsdata() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-newsdata/setup.py b/airbyte-integrations/connectors/source-newsdata/setup.py index a7bc19daa4ba..55b34a9fb708 100644 --- a/airbyte-integrations/connectors/source-newsdata/setup.py +++ b/airbyte-integrations/connectors/source-newsdata/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-newsdata=source_newsdata.run:run", + ], + }, name="source_newsdata", description="Source implementation for Newsdata.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-newsdata/source_newsdata/run.py b/airbyte-integrations/connectors/source-newsdata/source_newsdata/run.py new file mode 100644 index 000000000000..2cdae722bfb0 --- /dev/null +++ b/airbyte-integrations/connectors/source-newsdata/source_newsdata/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_newsdata import SourceNewsdata + + +def run(): + source = SourceNewsdata() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-notion/main.py b/airbyte-integrations/connectors/source-notion/main.py index dd188d547982..671d6cd692fa 100644 --- a/airbyte-integrations/connectors/source-notion/main.py +++ b/airbyte-integrations/connectors/source-notion/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_notion import SourceNotion +from source_notion.run import run if __name__ == "__main__": - source = SourceNotion() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-notion/setup.py b/airbyte-integrations/connectors/source-notion/setup.py index 80a1ed81533f..d31dfbbdeee3 100644 --- a/airbyte-integrations/connectors/source-notion/setup.py +++ b/airbyte-integrations/connectors/source-notion/setup.py @@ -18,6 +18,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-notion=source_notion.run:run", + ], + }, name="source_notion", description="Source implementation for Notion.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-notion/source_notion/run.py b/airbyte-integrations/connectors/source-notion/source_notion/run.py new file mode 100644 index 000000000000..df14df5ee9e4 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_notion import SourceNotion + + +def run(): + source = SourceNotion() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-nytimes/main.py b/airbyte-integrations/connectors/source-nytimes/main.py index b3e9d432ee28..ed513aa4248e 100644 --- a/airbyte-integrations/connectors/source-nytimes/main.py +++ b/airbyte-integrations/connectors/source-nytimes/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_nytimes import SourceNytimes +from source_nytimes.run import run if __name__ == "__main__": - source = SourceNytimes() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-nytimes/setup.py b/airbyte-integrations/connectors/source-nytimes/setup.py index ae57a1144c9f..6c0cc1d179b5 100644 --- a/airbyte-integrations/connectors/source-nytimes/setup.py +++ b/airbyte-integrations/connectors/source-nytimes/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-nytimes=source_nytimes.run:run", + ], + }, name="source_nytimes", description="Source implementation for Nytimes.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-nytimes/source_nytimes/run.py b/airbyte-integrations/connectors/source-nytimes/source_nytimes/run.py new file mode 100644 index 000000000000..ef4ddbd555e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-nytimes/source_nytimes/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_nytimes import SourceNytimes + + +def run(): + source = SourceNytimes() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-okta/main.py b/airbyte-integrations/connectors/source-okta/main.py index 93c853b816f5..488cb6ca1b6f 100644 --- a/airbyte-integrations/connectors/source-okta/main.py +++ b/airbyte-integrations/connectors/source-okta/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_okta import SourceOkta +from source_okta.run import run if __name__ == "__main__": - source = SourceOkta() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-okta/setup.py b/airbyte-integrations/connectors/source-okta/setup.py index a3fac533b39e..1a15dc58fd47 100644 --- a/airbyte-integrations/connectors/source-okta/setup.py +++ b/airbyte-integrations/connectors/source-okta/setup.py @@ -17,6 +17,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-okta=source_okta.run:run", + ], + }, name="source_okta", description="Source implementation for Okta.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-okta/source_okta/run.py b/airbyte-integrations/connectors/source-okta/source_okta/run.py new file mode 100644 index 000000000000..f40c87dc0e37 --- /dev/null +++ b/airbyte-integrations/connectors/source-okta/source_okta/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_okta import SourceOkta + + +def run(): + source = SourceOkta() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-omnisend/main.py b/airbyte-integrations/connectors/source-omnisend/main.py index 01c8cb75e133..bf2d072e4c00 100644 --- a/airbyte-integrations/connectors/source-omnisend/main.py +++ b/airbyte-integrations/connectors/source-omnisend/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_omnisend import SourceOmnisend +from source_omnisend.run import run if __name__ == "__main__": - source = SourceOmnisend() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-omnisend/setup.py b/airbyte-integrations/connectors/source-omnisend/setup.py index 653728e512b5..990506f80f9b 100644 --- a/airbyte-integrations/connectors/source-omnisend/setup.py +++ b/airbyte-integrations/connectors/source-omnisend/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-omnisend=source_omnisend.run:run", + ], + }, name="source_omnisend", description="Source implementation for Omnisend.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-omnisend/source_omnisend/run.py b/airbyte-integrations/connectors/source-omnisend/source_omnisend/run.py new file mode 100644 index 000000000000..46479c299f0a --- /dev/null +++ b/airbyte-integrations/connectors/source-omnisend/source_omnisend/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_omnisend import SourceOmnisend + + +def run(): + source = SourceOmnisend() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-onesignal/main.py b/airbyte-integrations/connectors/source-onesignal/main.py index ccff041b46e3..e735cf13bacb 100644 --- a/airbyte-integrations/connectors/source-onesignal/main.py +++ b/airbyte-integrations/connectors/source-onesignal/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_onesignal import SourceOnesignal +from source_onesignal.run import run if __name__ == "__main__": - source = SourceOnesignal() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-onesignal/setup.py b/airbyte-integrations/connectors/source-onesignal/setup.py index 47af71852c6a..aac6704b131b 100644 --- a/airbyte-integrations/connectors/source-onesignal/setup.py +++ b/airbyte-integrations/connectors/source-onesignal/setup.py @@ -12,13 +12,30 @@ TEST_REQUIREMENTS = ["pytest~=6.2", "pytest-mock~=3.6.1", "connector-acceptance-test"] setup( + entry_points={ + "console_scripts": [ + "source-onesignal=source_onesignal.run:run", + ], + }, name="source_onesignal", description="Source implementation for Onesignal.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-onesignal/source_onesignal/run.py b/airbyte-integrations/connectors/source-onesignal/source_onesignal/run.py new file mode 100644 index 000000000000..e5690d762c47 --- /dev/null +++ b/airbyte-integrations/connectors/source-onesignal/source_onesignal/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_onesignal import SourceOnesignal + + +def run(): + source = SourceOnesignal() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/main.py b/airbyte-integrations/connectors/source-open-exchange-rates/main.py index b311cb841f61..6223dbf6da8f 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/main.py +++ b/airbyte-integrations/connectors/source-open-exchange-rates/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_open_exchange_rates import SourceOpenExchangeRates +from source_open_exchange_rates.run import run if __name__ == "__main__": - source = SourceOpenExchangeRates() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/setup.py b/airbyte-integrations/connectors/source-open-exchange-rates/setup.py index 55fd4589213f..4b12738562df 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/setup.py +++ b/airbyte-integrations/connectors/source-open-exchange-rates/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-open-exchange-rates=source_open_exchange_rates.run:run", + ], + }, name="source_open_exchange_rates", description="Source implementation for Open Exchange Rates.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/run.py b/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/run.py new file mode 100644 index 000000000000..b2d83005c0ed --- /dev/null +++ b/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_open_exchange_rates import SourceOpenExchangeRates + + +def run(): + source = SourceOpenExchangeRates() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-openweather/main.py b/airbyte-integrations/connectors/source-openweather/main.py index 398948d54171..381fb38ae2dc 100644 --- a/airbyte-integrations/connectors/source-openweather/main.py +++ b/airbyte-integrations/connectors/source-openweather/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_openweather import SourceOpenweather +from source_openweather.run import run if __name__ == "__main__": - source = SourceOpenweather() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-openweather/setup.py b/airbyte-integrations/connectors/source-openweather/setup.py index fd32d0604449..2f978664b057 100644 --- a/airbyte-integrations/connectors/source-openweather/setup.py +++ b/airbyte-integrations/connectors/source-openweather/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-openweather=source_openweather.run:run", + ], + }, name="source_openweather", description="Source implementation for Openweather.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-openweather/source_openweather/run.py b/airbyte-integrations/connectors/source-openweather/source_openweather/run.py new file mode 100644 index 000000000000..f18939c72eb2 --- /dev/null +++ b/airbyte-integrations/connectors/source-openweather/source_openweather/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_openweather import SourceOpenweather + + +def run(): + source = SourceOpenweather() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-opsgenie/main.py b/airbyte-integrations/connectors/source-opsgenie/main.py index 0acd6dee136d..4f5798deffe1 100644 --- a/airbyte-integrations/connectors/source-opsgenie/main.py +++ b/airbyte-integrations/connectors/source-opsgenie/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_opsgenie import SourceOpsgenie +from source_opsgenie.run import run if __name__ == "__main__": - source = SourceOpsgenie() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-opsgenie/setup.py b/airbyte-integrations/connectors/source-opsgenie/setup.py index 48fa326b15b9..4bf256910f2b 100644 --- a/airbyte-integrations/connectors/source-opsgenie/setup.py +++ b/airbyte-integrations/connectors/source-opsgenie/setup.py @@ -15,13 +15,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-opsgenie=source_opsgenie.run:run", + ], + }, name="source_opsgenie", description="Source implementation for Opsgenie.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/run.py b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/run.py new file mode 100644 index 000000000000..63e0eb89c264 --- /dev/null +++ b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_opsgenie import SourceOpsgenie + + +def run(): + source = SourceOpsgenie() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-orb/main.py b/airbyte-integrations/connectors/source-orb/main.py index 08c8985a056a..725dc3553298 100644 --- a/airbyte-integrations/connectors/source-orb/main.py +++ b/airbyte-integrations/connectors/source-orb/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_orb import SourceOrb +from source_orb.run import run if __name__ == "__main__": - source = SourceOrb() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-orb/setup.py b/airbyte-integrations/connectors/source-orb/setup.py index 38e64fa19ee1..68f05af00a2c 100644 --- a/airbyte-integrations/connectors/source-orb/setup.py +++ b/airbyte-integrations/connectors/source-orb/setup.py @@ -10,6 +10,11 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", "responses~=0.13.3", "pendulum==2.1.2"] setup( + entry_points={ + "console_scripts": [ + "source-orb=source_orb.run:run", + ], + }, name="source_orb", description="Source implementation for Orb.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-orb/source_orb/run.py b/airbyte-integrations/connectors/source-orb/source_orb/run.py new file mode 100644 index 000000000000..05f19468aac9 --- /dev/null +++ b/airbyte-integrations/connectors/source-orb/source_orb/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_orb import SourceOrb + + +def run(): + source = SourceOrb() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-orbit/main.py b/airbyte-integrations/connectors/source-orbit/main.py index 82e25dd0caa5..16b6891a52b6 100644 --- a/airbyte-integrations/connectors/source-orbit/main.py +++ b/airbyte-integrations/connectors/source-orbit/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_orbit import SourceOrbit +from source_orbit.run import run if __name__ == "__main__": - source = SourceOrbit() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-orbit/setup.py b/airbyte-integrations/connectors/source-orbit/setup.py index ae8d7e9b1c8b..9ca2a1394b2a 100644 --- a/airbyte-integrations/connectors/source-orbit/setup.py +++ b/airbyte-integrations/connectors/source-orbit/setup.py @@ -10,13 +10,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-orbit=source_orbit.run:run", + ], + }, name="source_orbit", description="Source implementation for Orbit.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-orbit/source_orbit/run.py b/airbyte-integrations/connectors/source-orbit/source_orbit/run.py new file mode 100644 index 000000000000..593cfb9a326a --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/source_orbit/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_orbit import SourceOrbit + + +def run(): + source = SourceOrbit() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-oura/main.py b/airbyte-integrations/connectors/source-oura/main.py index 35f15399d958..f72173126bd0 100644 --- a/airbyte-integrations/connectors/source-oura/main.py +++ b/airbyte-integrations/connectors/source-oura/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_oura import SourceOura +from source_oura.run import run if __name__ == "__main__": - source = SourceOura() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-oura/setup.py b/airbyte-integrations/connectors/source-oura/setup.py index b78b256c87fb..36e29e32605c 100644 --- a/airbyte-integrations/connectors/source-oura/setup.py +++ b/airbyte-integrations/connectors/source-oura/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-oura=source_oura.run:run", + ], + }, name="source_oura", description="Source implementation for Oura.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-oura/source_oura/run.py b/airbyte-integrations/connectors/source-oura/source_oura/run.py new file mode 100644 index 000000000000..af46311895fb --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/source_oura/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_oura import SourceOura + + +def run(): + source = SourceOura() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/main.py b/airbyte-integrations/connectors/source-outbrain-amplify/main.py index 2f2acbc0b627..dc58d39d5bcd 100644 --- a/airbyte-integrations/connectors/source-outbrain-amplify/main.py +++ b/airbyte-integrations/connectors/source-outbrain-amplify/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_outbrain_amplify import SourceOutbrainAmplify +from source_outbrain_amplify.run import run if __name__ == "__main__": - source = SourceOutbrainAmplify() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/setup.py b/airbyte-integrations/connectors/source-outbrain-amplify/setup.py index d8b5563f3dce..f4f709b99a49 100644 --- a/airbyte-integrations/connectors/source-outbrain-amplify/setup.py +++ b/airbyte-integrations/connectors/source-outbrain-amplify/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-outbrain-amplify=source_outbrain_amplify.run:run", + ], + }, name="source_outbrain_amplify", description="Source implementation for Outbrain Amplify.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/run.py b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/run.py new file mode 100644 index 000000000000..8f90ae8e0469 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_outbrain_amplify import SourceOutbrainAmplify + + +def run(): + source = SourceOutbrainAmplify() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-outreach/main.py b/airbyte-integrations/connectors/source-outreach/main.py index 77394c375d03..9ad460e89200 100644 --- a/airbyte-integrations/connectors/source-outreach/main.py +++ b/airbyte-integrations/connectors/source-outreach/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_outreach import SourceOutreach +from source_outreach.run import run if __name__ == "__main__": - source = SourceOutreach() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-outreach/setup.py b/airbyte-integrations/connectors/source-outreach/setup.py index 2367148b8d73..dc15e76330f8 100644 --- a/airbyte-integrations/connectors/source-outreach/setup.py +++ b/airbyte-integrations/connectors/source-outreach/setup.py @@ -16,6 +16,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-outreach=source_outreach.run:run", + ], + }, name="source_outreach", description="Source implementation for Outreach.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/run.py b/airbyte-integrations/connectors/source-outreach/source_outreach/run.py new file mode 100644 index 000000000000..2d48fe0806cd --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_outreach import SourceOutreach + + +def run(): + source = SourceOutreach() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pagerduty/main.py b/airbyte-integrations/connectors/source-pagerduty/main.py index 61d193268f6b..22537946cc8c 100644 --- a/airbyte-integrations/connectors/source-pagerduty/main.py +++ b/airbyte-integrations/connectors/source-pagerduty/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_pagerduty import SourcePagerduty +from source_pagerduty.run import run if __name__ == "__main__": - source = SourcePagerduty() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-pagerduty/setup.py b/airbyte-integrations/connectors/source-pagerduty/setup.py index aec396ac036d..cba7caa9d5bd 100644 --- a/airbyte-integrations/connectors/source-pagerduty/setup.py +++ b/airbyte-integrations/connectors/source-pagerduty/setup.py @@ -14,13 +14,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-pagerduty=source_pagerduty.run:run", + ], + }, name="source_pagerduty", description="Source implementation for Pagerduty.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/run.py b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/run.py new file mode 100644 index 000000000000..f73afe9c6592 --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pagerduty import SourcePagerduty + + +def run(): + source = SourcePagerduty() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pardot/main.py b/airbyte-integrations/connectors/source-pardot/main.py index 8158c5fc7d66..6112e4e13e27 100644 --- a/airbyte-integrations/connectors/source-pardot/main.py +++ b/airbyte-integrations/connectors/source-pardot/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_pardot import SourcePardot +from source_pardot.run import run if __name__ == "__main__": - source = SourcePardot() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-pardot/setup.py b/airbyte-integrations/connectors/source-pardot/setup.py index c04454b0bd05..47cbea1de2c0 100644 --- a/airbyte-integrations/connectors/source-pardot/setup.py +++ b/airbyte-integrations/connectors/source-pardot/setup.py @@ -16,6 +16,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-pardot=source_pardot.run:run", + ], + }, name="source_pardot", description="Source implementation for Pardot.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-pardot/source_pardot/run.py b/airbyte-integrations/connectors/source-pardot/source_pardot/run.py new file mode 100644 index 000000000000..34bd6425a06d --- /dev/null +++ b/airbyte-integrations/connectors/source-pardot/source_pardot/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pardot import SourcePardot + + +def run(): + source = SourcePardot() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-partnerstack/main.py b/airbyte-integrations/connectors/source-partnerstack/main.py index 24bd01814100..d22642a3ee66 100644 --- a/airbyte-integrations/connectors/source-partnerstack/main.py +++ b/airbyte-integrations/connectors/source-partnerstack/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_partnerstack import SourcePartnerstack +from source_partnerstack.run import run if __name__ == "__main__": - source = SourcePartnerstack() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-partnerstack/setup.py b/airbyte-integrations/connectors/source-partnerstack/setup.py index 5e6542bbc4d6..4d13b3c870f8 100644 --- a/airbyte-integrations/connectors/source-partnerstack/setup.py +++ b/airbyte-integrations/connectors/source-partnerstack/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-partnerstack=source_partnerstack.run:run", + ], + }, name="source_partnerstack", description="Source implementation for Partnerstack.", author="Elliot Trabac", author_email="elliot.trabac1@gmail.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/run.py b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/run.py new file mode 100644 index 000000000000..1b7dad130b55 --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_partnerstack import SourcePartnerstack + + +def run(): + source = SourcePartnerstack() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/setup.py b/airbyte-integrations/connectors/source-paypal-transaction/setup.py index a7f633e8ca28..1fdb3b7826a2 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/setup.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/setup.py @@ -27,7 +27,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-paystack/main.py b/airbyte-integrations/connectors/source-paystack/main.py index 04c982f1ed90..8f792800f6de 100644 --- a/airbyte-integrations/connectors/source-paystack/main.py +++ b/airbyte-integrations/connectors/source-paystack/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_paystack import SourcePaystack +from source_paystack.run import run if __name__ == "__main__": - source = SourcePaystack() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-paystack/setup.py b/airbyte-integrations/connectors/source-paystack/setup.py index 946504314000..22f2c755d169 100644 --- a/airbyte-integrations/connectors/source-paystack/setup.py +++ b/airbyte-integrations/connectors/source-paystack/setup.py @@ -12,6 +12,11 @@ TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6.1", "requests-mock"] setup( + entry_points={ + "console_scripts": [ + "source-paystack=source_paystack.run:run", + ], + }, name="source_paystack", description="Source implementation for Paystack.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/run.py b/airbyte-integrations/connectors/source-paystack/source_paystack/run.py new file mode 100644 index 000000000000..001e5668317e --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_paystack import SourcePaystack + + +def run(): + source = SourcePaystack() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pendo/main.py b/airbyte-integrations/connectors/source-pendo/main.py index 6d17f4106c1d..882482d947f8 100644 --- a/airbyte-integrations/connectors/source-pendo/main.py +++ b/airbyte-integrations/connectors/source-pendo/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_pendo import SourcePendo +from source_pendo.run import run if __name__ == "__main__": - source = SourcePendo() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-pendo/setup.py b/airbyte-integrations/connectors/source-pendo/setup.py index 8aa15183ac48..907aa867ff7b 100644 --- a/airbyte-integrations/connectors/source-pendo/setup.py +++ b/airbyte-integrations/connectors/source-pendo/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-pendo=source_pendo.run:run", + ], + }, name="source_pendo", description="Source implementation for Pendo.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pendo/source_pendo/run.py b/airbyte-integrations/connectors/source-pendo/source_pendo/run.py new file mode 100644 index 000000000000..a98bdc7d16ff --- /dev/null +++ b/airbyte-integrations/connectors/source-pendo/source_pendo/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pendo import SourcePendo + + +def run(): + source = SourcePendo() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-persistiq/main.py b/airbyte-integrations/connectors/source-persistiq/main.py index 438306b66706..b6be0062f615 100644 --- a/airbyte-integrations/connectors/source-persistiq/main.py +++ b/airbyte-integrations/connectors/source-persistiq/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_persistiq import SourcePersistiq +from source_persistiq.run import run if __name__ == "__main__": - source = SourcePersistiq() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-persistiq/setup.py b/airbyte-integrations/connectors/source-persistiq/setup.py index 88f883fbdacf..509b4ddc1797 100644 --- a/airbyte-integrations/connectors/source-persistiq/setup.py +++ b/airbyte-integrations/connectors/source-persistiq/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-persistiq=source_persistiq.run:run", + ], + }, name="source_persistiq", description="Source implementation for Persistiq.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-persistiq/source_persistiq/run.py b/airbyte-integrations/connectors/source-persistiq/source_persistiq/run.py new file mode 100644 index 000000000000..2f604abf6ae5 --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/source_persistiq/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_persistiq import SourcePersistiq + + +def run(): + source = SourcePersistiq() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pexels-api/main.py b/airbyte-integrations/connectors/source-pexels-api/main.py index f13d88a9f301..4fdc405e120e 100644 --- a/airbyte-integrations/connectors/source-pexels-api/main.py +++ b/airbyte-integrations/connectors/source-pexels-api/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_pexels_api import SourcePexelsApi +from source_pexels_api.run import run if __name__ == "__main__": - source = SourcePexelsApi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-pexels-api/setup.py b/airbyte-integrations/connectors/source-pexels-api/setup.py index e45f600af446..7525bf7113f5 100644 --- a/airbyte-integrations/connectors/source-pexels-api/setup.py +++ b/airbyte-integrations/connectors/source-pexels-api/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-pexels-api=source_pexels_api.run:run", + ], + }, name="source_pexels_api", description="Source implementation for Pexels Api.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/run.py b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/run.py new file mode 100644 index 000000000000..da128a40cea0 --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pexels_api import SourcePexelsApi + + +def run(): + source = SourcePexelsApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pinterest/main.py b/airbyte-integrations/connectors/source-pinterest/main.py index 377a6ad6dd76..aff013c70319 100644 --- a/airbyte-integrations/connectors/source-pinterest/main.py +++ b/airbyte-integrations/connectors/source-pinterest/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_pinterest import SourcePinterest +from source_pinterest.run import run if __name__ == "__main__": - source = SourcePinterest() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-pinterest/setup.py b/airbyte-integrations/connectors/source-pinterest/setup.py index 5da646d8e719..d09828b5c17e 100644 --- a/airbyte-integrations/connectors/source-pinterest/setup.py +++ b/airbyte-integrations/connectors/source-pinterest/setup.py @@ -15,6 +15,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-pinterest=source_pinterest.run:run", + ], + }, name="source_pinterest", description="Source implementation for Pinterest.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/run.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/run.py new file mode 100644 index 000000000000..2acf5886cb16 --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pinterest import SourcePinterest + + +def run(): + source = SourcePinterest() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pipedrive/setup.py b/airbyte-integrations/connectors/source-pipedrive/setup.py index 5d3c1999b80c..388dc3e91906 100644 --- a/airbyte-integrations/connectors/source-pipedrive/setup.py +++ b/airbyte-integrations/connectors/source-pipedrive/setup.py @@ -26,7 +26,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/main.py b/airbyte-integrations/connectors/source-pivotal-tracker/main.py index 3115bba2f932..767718108296 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/main.py +++ b/airbyte-integrations/connectors/source-pivotal-tracker/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_pivotal_tracker import SourcePivotalTracker +from source_pivotal_tracker.run import run if __name__ == "__main__": - source = SourcePivotalTracker() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/setup.py b/airbyte-integrations/connectors/source-pivotal-tracker/setup.py index 63de73cbd36f..e6c5edb752f1 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/setup.py +++ b/airbyte-integrations/connectors/source-pivotal-tracker/setup.py @@ -17,6 +17,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-pivotal-tracker=source_pivotal_tracker.run:run", + ], + }, name="source_pivotal_tracker", description="Source implementation for Pivotal Tracker.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/run.py b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/run.py new file mode 100644 index 000000000000..a187ff169ac2 --- /dev/null +++ b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pivotal_tracker import SourcePivotalTracker + + +def run(): + source = SourcePivotalTracker() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-plaid/main.py b/airbyte-integrations/connectors/source-plaid/main.py index e8f3663ddccc..be0011cc4860 100644 --- a/airbyte-integrations/connectors/source-plaid/main.py +++ b/airbyte-integrations/connectors/source-plaid/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_plaid import SourcePlaid +from source_plaid.run import run if __name__ == "__main__": - source = SourcePlaid() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-plaid/setup.py b/airbyte-integrations/connectors/source-plaid/setup.py index bcbf0b34a80c..fc6368795555 100644 --- a/airbyte-integrations/connectors/source-plaid/setup.py +++ b/airbyte-integrations/connectors/source-plaid/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-plaid=source_plaid.run:run", + ], + }, name="source_plaid", description="Source implementation for Plaid.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-plaid/source_plaid/run.py b/airbyte-integrations/connectors/source-plaid/source_plaid/run.py new file mode 100644 index 000000000000..c83935220787 --- /dev/null +++ b/airbyte-integrations/connectors/source-plaid/source_plaid/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_plaid import SourcePlaid + + +def run(): + source = SourcePlaid() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-plausible/main.py b/airbyte-integrations/connectors/source-plausible/main.py index 581156e36e0f..453a2f457795 100644 --- a/airbyte-integrations/connectors/source-plausible/main.py +++ b/airbyte-integrations/connectors/source-plausible/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_plausible import SourcePlausible +from source_plausible.run import run if __name__ == "__main__": - source = SourcePlausible() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-plausible/setup.py b/airbyte-integrations/connectors/source-plausible/setup.py index d222145d848e..894526785513 100644 --- a/airbyte-integrations/connectors/source-plausible/setup.py +++ b/airbyte-integrations/connectors/source-plausible/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-plausible=source_plausible.run:run", + ], + }, name="source_plausible", description="Source implementation for Plausible.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-plausible/source_plausible/run.py b/airbyte-integrations/connectors/source-plausible/source_plausible/run.py new file mode 100644 index 000000000000..57370a5f3748 --- /dev/null +++ b/airbyte-integrations/connectors/source-plausible/source_plausible/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_plausible import SourcePlausible + + +def run(): + source = SourcePlausible() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pocket/main.py b/airbyte-integrations/connectors/source-pocket/main.py index bf2c5d41e121..45d6104c34f8 100644 --- a/airbyte-integrations/connectors/source-pocket/main.py +++ b/airbyte-integrations/connectors/source-pocket/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_pocket import SourcePocket +from source_pocket.run import run if __name__ == "__main__": - source = SourcePocket() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-pocket/setup.py b/airbyte-integrations/connectors/source-pocket/setup.py index bdbf2b6549ab..d7238d34a09e 100644 --- a/airbyte-integrations/connectors/source-pocket/setup.py +++ b/airbyte-integrations/connectors/source-pocket/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-pocket=source_pocket.run:run", + ], + }, name="source_pocket", description="Source implementation for Pocket.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pocket/source_pocket/run.py b/airbyte-integrations/connectors/source-pocket/source_pocket/run.py new file mode 100644 index 000000000000..75eaea5398f6 --- /dev/null +++ b/airbyte-integrations/connectors/source-pocket/source_pocket/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pocket import SourcePocket + + +def run(): + source = SourcePocket() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pokeapi/main.py b/airbyte-integrations/connectors/source-pokeapi/main.py index 38a510a3f2d7..f32ce6b38105 100644 --- a/airbyte-integrations/connectors/source-pokeapi/main.py +++ b/airbyte-integrations/connectors/source-pokeapi/main.py @@ -2,11 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import sys - -from airbyte_cdk.entrypoint import launch -from source_pokeapi import SourcePokeapi +from source_pokeapi.run import run if __name__ == "__main__": - source = SourcePokeapi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-pokeapi/setup.py b/airbyte-integrations/connectors/source-pokeapi/setup.py index 2fa7839b58fc..27a5590ece4d 100644 --- a/airbyte-integrations/connectors/source-pokeapi/setup.py +++ b/airbyte-integrations/connectors/source-pokeapi/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-pokeapi=source_pokeapi.run:run", + ], + }, name="source_pokeapi", description="Source implementation for Pokeapi.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/run.py b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/run.py new file mode 100644 index 000000000000..2b573e693954 --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/run.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pokeapi import SourcePokeapi + + +def run(): + source = SourcePokeapi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-polygon-stock-api/main.py b/airbyte-integrations/connectors/source-polygon-stock-api/main.py index 7fc7ae2dd497..77c0f73430be 100644 --- a/airbyte-integrations/connectors/source-polygon-stock-api/main.py +++ b/airbyte-integrations/connectors/source-polygon-stock-api/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_polygon_stock_api import SourcePolygonStockApi +from source_polygon_stock_api.run import run if __name__ == "__main__": - source = SourcePolygonStockApi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-polygon-stock-api/setup.py b/airbyte-integrations/connectors/source-polygon-stock-api/setup.py index ddd7463c9292..2b4930c184d3 100644 --- a/airbyte-integrations/connectors/source-polygon-stock-api/setup.py +++ b/airbyte-integrations/connectors/source-polygon-stock-api/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-polygon-stock-api=source_polygon_stock_api.run:run", + ], + }, name="source_polygon_stock_api", description="Source implementation for Polygon Stock Api.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-polygon-stock-api/source_polygon_stock_api/run.py b/airbyte-integrations/connectors/source-polygon-stock-api/source_polygon_stock_api/run.py new file mode 100644 index 000000000000..ab2a75d41fe6 --- /dev/null +++ b/airbyte-integrations/connectors/source-polygon-stock-api/source_polygon_stock_api/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_polygon_stock_api import SourcePolygonStockApi + + +def run(): + source = SourcePolygonStockApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-posthog/main.py b/airbyte-integrations/connectors/source-posthog/main.py index c788205d69b6..f7e69357d999 100644 --- a/airbyte-integrations/connectors/source-posthog/main.py +++ b/airbyte-integrations/connectors/source-posthog/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_posthog import SourcePosthog +from source_posthog.run import run if __name__ == "__main__": - source = SourcePosthog() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-posthog/setup.py b/airbyte-integrations/connectors/source-posthog/setup.py index 75fb93d9ac2c..2f23e21f1cf8 100644 --- a/airbyte-integrations/connectors/source-posthog/setup.py +++ b/airbyte-integrations/connectors/source-posthog/setup.py @@ -14,6 +14,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-posthog=source_posthog.run:run", + ], + }, name="source_posthog", description="Source implementation for Posthog.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/run.py b/airbyte-integrations/connectors/source-posthog/source_posthog/run.py new file mode 100644 index 000000000000..711b2e4ae483 --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_posthog import SourcePosthog + + +def run(): + source = SourcePosthog() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-postmarkapp/main.py b/airbyte-integrations/connectors/source-postmarkapp/main.py index 8257a82e80c1..c6b256d74b3a 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/main.py +++ b/airbyte-integrations/connectors/source-postmarkapp/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_postmarkapp import SourcePostmarkapp +from source_postmarkapp.run import run if __name__ == "__main__": - source = SourcePostmarkapp() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-postmarkapp/setup.py b/airbyte-integrations/connectors/source-postmarkapp/setup.py index 08a989882942..3c5e3ab4e671 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/setup.py +++ b/airbyte-integrations/connectors/source-postmarkapp/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-postmarkapp=source_postmarkapp.run:run", + ], + }, name="source_postmarkapp", description="Source implementation for Postmarkapp.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/run.py b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/run.py new file mode 100644 index 000000000000..b3308b251000 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_postmarkapp import SourcePostmarkapp + + +def run(): + source = SourcePostmarkapp() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-prestashop/main.py b/airbyte-integrations/connectors/source-prestashop/main.py index 813f7411045f..e51c47a996aa 100644 --- a/airbyte-integrations/connectors/source-prestashop/main.py +++ b/airbyte-integrations/connectors/source-prestashop/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_prestashop import SourcePrestashop +from source_prestashop.run import run if __name__ == "__main__": - source = SourcePrestashop() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-prestashop/setup.py b/airbyte-integrations/connectors/source-prestashop/setup.py index 4762aa8a2eaf..e883c046c0af 100644 --- a/airbyte-integrations/connectors/source-prestashop/setup.py +++ b/airbyte-integrations/connectors/source-prestashop/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-prestashop=source_prestashop.run:run", + ], + }, name="source_prestashop", description="Source implementation for PrestaShop.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/run.py b/airbyte-integrations/connectors/source-prestashop/source_prestashop/run.py new file mode 100644 index 000000000000..23f77c77bf55 --- /dev/null +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_prestashop import SourcePrestashop + + +def run(): + source = SourcePrestashop() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-primetric/main.py b/airbyte-integrations/connectors/source-primetric/main.py index 3cbd6368b2cf..3cb9977197cc 100644 --- a/airbyte-integrations/connectors/source-primetric/main.py +++ b/airbyte-integrations/connectors/source-primetric/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_primetric import SourcePrimetric +from source_primetric.run import run if __name__ == "__main__": - source = SourcePrimetric() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-primetric/setup.py b/airbyte-integrations/connectors/source-primetric/setup.py index 00a6f81dc6b3..d73e7ee77688 100644 --- a/airbyte-integrations/connectors/source-primetric/setup.py +++ b/airbyte-integrations/connectors/source-primetric/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-primetric=source_primetric.run:run", + ], + }, name="source_primetric", description="Source implementation for Primetric.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-primetric/source_primetric/run.py b/airbyte-integrations/connectors/source-primetric/source_primetric/run.py new file mode 100644 index 000000000000..729752166581 --- /dev/null +++ b/airbyte-integrations/connectors/source-primetric/source_primetric/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_primetric import SourcePrimetric + + +def run(): + source = SourcePrimetric() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-public-apis/main.py b/airbyte-integrations/connectors/source-public-apis/main.py index c5ad4ac06bd9..c9796a4aa4ef 100644 --- a/airbyte-integrations/connectors/source-public-apis/main.py +++ b/airbyte-integrations/connectors/source-public-apis/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_public_apis import SourcePublicApis +from source_public_apis.run import run if __name__ == "__main__": - source = SourcePublicApis() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-public-apis/setup.py b/airbyte-integrations/connectors/source-public-apis/setup.py index 8e3f218e28f7..d454fb726a25 100644 --- a/airbyte-integrations/connectors/source-public-apis/setup.py +++ b/airbyte-integrations/connectors/source-public-apis/setup.py @@ -12,13 +12,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-public-apis=source_public_apis.run:run", + ], + }, name="source_public_apis", description="Source implementation for Public Apis.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-public-apis/source_public_apis/run.py b/airbyte-integrations/connectors/source-public-apis/source_public_apis/run.py new file mode 100644 index 000000000000..b4927fb5a5e2 --- /dev/null +++ b/airbyte-integrations/connectors/source-public-apis/source_public_apis/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_public_apis import SourcePublicApis + + +def run(): + source = SourcePublicApis() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-punk-api/main.py b/airbyte-integrations/connectors/source-punk-api/main.py index c16f1cc798dc..bd3661bdabfc 100644 --- a/airbyte-integrations/connectors/source-punk-api/main.py +++ b/airbyte-integrations/connectors/source-punk-api/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_punk_api import SourcePunkApi +from source_punk_api.run import run if __name__ == "__main__": - source = SourcePunkApi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-punk-api/setup.py b/airbyte-integrations/connectors/source-punk-api/setup.py index 6e0119b0fd17..834098b436fe 100644 --- a/airbyte-integrations/connectors/source-punk-api/setup.py +++ b/airbyte-integrations/connectors/source-punk-api/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-punk-api=source_punk_api.run:run", + ], + }, name="source_punk_api", description="Source implementation for Punk Api.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-punk-api/source_punk_api/run.py b/airbyte-integrations/connectors/source-punk-api/source_punk_api/run.py new file mode 100644 index 000000000000..f1a31bfed3f7 --- /dev/null +++ b/airbyte-integrations/connectors/source-punk-api/source_punk_api/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_punk_api import SourcePunkApi + + +def run(): + source = SourcePunkApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pypi/main.py b/airbyte-integrations/connectors/source-pypi/main.py index a61df9711f41..44f5f367986f 100644 --- a/airbyte-integrations/connectors/source-pypi/main.py +++ b/airbyte-integrations/connectors/source-pypi/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_pypi import SourcePypi +from source_pypi.run import run if __name__ == "__main__": - source = SourcePypi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-pypi/setup.py b/airbyte-integrations/connectors/source-pypi/setup.py index ae9a89a654a5..b1198fecaead 100644 --- a/airbyte-integrations/connectors/source-pypi/setup.py +++ b/airbyte-integrations/connectors/source-pypi/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-pypi=source_pypi.run:run", + ], + }, name="source_pypi", description="Source implementation for Pypi.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pypi/source_pypi/run.py b/airbyte-integrations/connectors/source-pypi/source_pypi/run.py new file mode 100644 index 000000000000..5fc71c441a59 --- /dev/null +++ b/airbyte-integrations/connectors/source-pypi/source_pypi/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pypi import SourcePypi + + +def run(): + source = SourcePypi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-python-http-tutorial/main.py b/airbyte-integrations/connectors/source-python-http-tutorial/main.py index e13b685ed7d4..57dce4e0679c 100644 --- a/airbyte-integrations/connectors/source-python-http-tutorial/main.py +++ b/airbyte-integrations/connectors/source-python-http-tutorial/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_python_http_tutorial import SourcePythonHttpTutorial +from source_python_http_tutorial.run import run if __name__ == "__main__": - source = SourcePythonHttpTutorial() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-python-http-tutorial/setup.py b/airbyte-integrations/connectors/source-python-http-tutorial/setup.py index 473e22de73aa..be40ba5ca6c6 100644 --- a/airbyte-integrations/connectors/source-python-http-tutorial/setup.py +++ b/airbyte-integrations/connectors/source-python-http-tutorial/setup.py @@ -5,6 +5,11 @@ from setuptools import find_packages, setup setup( + entry_points={ + "console_scripts": [ + "source-python-http-tutorial=source_python_http_tutorial.run:run", + ], + }, name="source_python_http_tutorial", description="Source implementation for Python Http Tutorial.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/run.py b/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/run.py new file mode 100644 index 000000000000..60690aa68eff --- /dev/null +++ b/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_python_http_tutorial import SourcePythonHttpTutorial + + +def run(): + source = SourcePythonHttpTutorial() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-qonto/main.py b/airbyte-integrations/connectors/source-qonto/main.py index 749ae981779f..eb2e90e366c1 100644 --- a/airbyte-integrations/connectors/source-qonto/main.py +++ b/airbyte-integrations/connectors/source-qonto/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_qonto import SourceQonto +from source_qonto.run import run if __name__ == "__main__": - source = SourceQonto() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-qonto/setup.py b/airbyte-integrations/connectors/source-qonto/setup.py index 8847f9443761..ba340d0b242a 100644 --- a/airbyte-integrations/connectors/source-qonto/setup.py +++ b/airbyte-integrations/connectors/source-qonto/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-qonto=source_qonto.run:run", + ], + }, name="source_qonto", description="Source implementation for Qonto.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-qonto/source_qonto/run.py b/airbyte-integrations/connectors/source-qonto/source_qonto/run.py new file mode 100644 index 000000000000..0ea8c8470c1f --- /dev/null +++ b/airbyte-integrations/connectors/source-qonto/source_qonto/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_qonto import SourceQonto + + +def run(): + source = SourceQonto() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-qualaroo/main.py b/airbyte-integrations/connectors/source-qualaroo/main.py index 149d1ab5a20f..7a7ed24a96bf 100644 --- a/airbyte-integrations/connectors/source-qualaroo/main.py +++ b/airbyte-integrations/connectors/source-qualaroo/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_qualaroo import SourceQualaroo +from source_qualaroo.run import run if __name__ == "__main__": - source = SourceQualaroo() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-qualaroo/setup.py b/airbyte-integrations/connectors/source-qualaroo/setup.py index 840126cac8fb..24bf8b993372 100644 --- a/airbyte-integrations/connectors/source-qualaroo/setup.py +++ b/airbyte-integrations/connectors/source-qualaroo/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-qualaroo=source_qualaroo.run:run", + ], + }, name="source_qualaroo", description="Source implementation for Qualaroo.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/run.py b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/run.py new file mode 100644 index 000000000000..c6b4c65009e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_qualaroo import SourceQualaroo + + +def run(): + source = SourceQualaroo() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-quickbooks/main.py b/airbyte-integrations/connectors/source-quickbooks/main.py index c475c3cf97fd..abeed13585f5 100644 --- a/airbyte-integrations/connectors/source-quickbooks/main.py +++ b/airbyte-integrations/connectors/source-quickbooks/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_quickbooks import SourceQuickbooks +from source_quickbooks.run import run if __name__ == "__main__": - source = SourceQuickbooks() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-quickbooks/setup.py b/airbyte-integrations/connectors/source-quickbooks/setup.py index 47ca1b128505..ac3c35cb290f 100644 --- a/airbyte-integrations/connectors/source-quickbooks/setup.py +++ b/airbyte-integrations/connectors/source-quickbooks/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-quickbooks=source_quickbooks.run:run", + ], + }, name="source_quickbooks", description="Source implementation for Quickbooks.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/run.py b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/run.py new file mode 100644 index 000000000000..4e5d18504e8d --- /dev/null +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_quickbooks import SourceQuickbooks + + +def run(): + source = SourceQuickbooks() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-railz/main.py b/airbyte-integrations/connectors/source-railz/main.py index 03bea5dbba8a..bfa6b9fadb2f 100644 --- a/airbyte-integrations/connectors/source-railz/main.py +++ b/airbyte-integrations/connectors/source-railz/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_railz import SourceRailz +from source_railz.run import run if __name__ == "__main__": - source = SourceRailz() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-railz/setup.py b/airbyte-integrations/connectors/source-railz/setup.py index 3b938dc45ff5..37478efb8ddc 100644 --- a/airbyte-integrations/connectors/source-railz/setup.py +++ b/airbyte-integrations/connectors/source-railz/setup.py @@ -19,13 +19,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-railz=source_railz.run:run", + ], + }, name="source_railz", description="Source implementation for Railz.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-railz/source_railz/run.py b/airbyte-integrations/connectors/source-railz/source_railz/run.py new file mode 100644 index 000000000000..831b665a05da --- /dev/null +++ b/airbyte-integrations/connectors/source-railz/source_railz/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_railz import SourceRailz + + +def run(): + source = SourceRailz() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/main.py b/airbyte-integrations/connectors/source-rd-station-marketing/main.py index b5e333d8a220..5210a9fdc2b3 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/main.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_rd_station_marketing import SourceRDStationMarketing +from source_rd_station_marketing.run import run if __name__ == "__main__": - source = SourceRDStationMarketing() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/setup.py b/airbyte-integrations/connectors/source-rd-station-marketing/setup.py index fc5481e12eb5..039dd82b7d54 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/setup.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-rd-station-marketing=source_rd_station_marketing.run:run", + ], + }, name="source_rd_station_marketing", description="Source implementation for RD Station Marketing.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/run.py b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/run.py new file mode 100644 index 000000000000..c684a12ed29f --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_rd_station_marketing import SourceRDStationMarketing + + +def run(): + source = SourceRDStationMarketing() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-recharge/main.py b/airbyte-integrations/connectors/source-recharge/main.py index c61ef445b68d..d8ccf40b711e 100644 --- a/airbyte-integrations/connectors/source-recharge/main.py +++ b/airbyte-integrations/connectors/source-recharge/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_recharge import SourceRecharge +from source_recharge.run import run if __name__ == "__main__": - source = SourceRecharge() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-recharge/setup.py b/airbyte-integrations/connectors/source-recharge/setup.py index bb091a439b28..d51c828ffa5b 100644 --- a/airbyte-integrations/connectors/source-recharge/setup.py +++ b/airbyte-integrations/connectors/source-recharge/setup.py @@ -16,6 +16,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-recharge=source_recharge.run:run", + ], + }, name="source_recharge", description="Source implementation for Recharge.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/run.py b/airbyte-integrations/connectors/source-recharge/source_recharge/run.py new file mode 100644 index 000000000000..2a56566c41c2 --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_recharge import SourceRecharge + + +def run(): + source = SourceRecharge() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-recreation/main.py b/airbyte-integrations/connectors/source-recreation/main.py index f126b44607d6..6847e93aab46 100644 --- a/airbyte-integrations/connectors/source-recreation/main.py +++ b/airbyte-integrations/connectors/source-recreation/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_recreation import SourceRecreation +from source_recreation.run import run if __name__ == "__main__": - source = SourceRecreation() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-recreation/setup.py b/airbyte-integrations/connectors/source-recreation/setup.py index 26e4b5657f8a..9f41baeed4b1 100644 --- a/airbyte-integrations/connectors/source-recreation/setup.py +++ b/airbyte-integrations/connectors/source-recreation/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-recreation=source_recreation.run:run", + ], + }, name="source_recreation", description="Source implementation for Recreation.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-recreation/source_recreation/run.py b/airbyte-integrations/connectors/source-recreation/source_recreation/run.py new file mode 100644 index 000000000000..2f8b496ad695 --- /dev/null +++ b/airbyte-integrations/connectors/source-recreation/source_recreation/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_recreation import SourceRecreation + + +def run(): + source = SourceRecreation() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-recruitee/main.py b/airbyte-integrations/connectors/source-recruitee/main.py index 8d918eec10e2..26802f658cda 100644 --- a/airbyte-integrations/connectors/source-recruitee/main.py +++ b/airbyte-integrations/connectors/source-recruitee/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_recruitee import SourceRecruitee +from source_recruitee.run import run if __name__ == "__main__": - source = SourceRecruitee() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-recruitee/setup.py b/airbyte-integrations/connectors/source-recruitee/setup.py index d2b12eff3c68..c03d7631bd28 100644 --- a/airbyte-integrations/connectors/source-recruitee/setup.py +++ b/airbyte-integrations/connectors/source-recruitee/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-recruitee=source_recruitee.run:run", + ], + }, name="source_recruitee", description="Source implementation for Recruitee.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-recruitee/source_recruitee/run.py b/airbyte-integrations/connectors/source-recruitee/source_recruitee/run.py new file mode 100644 index 000000000000..3d4111901af2 --- /dev/null +++ b/airbyte-integrations/connectors/source-recruitee/source_recruitee/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_recruitee import SourceRecruitee + + +def run(): + source = SourceRecruitee() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-recurly/main.py b/airbyte-integrations/connectors/source-recurly/main.py index 251a30f0dffe..ba5c26176fde 100644 --- a/airbyte-integrations/connectors/source-recurly/main.py +++ b/airbyte-integrations/connectors/source-recurly/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_recurly import SourceRecurly +from source_recurly.run import run if __name__ == "__main__": - source = SourceRecurly() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-recurly/setup.py b/airbyte-integrations/connectors/source-recurly/setup.py index 1d278d40ec9f..16e2364aeb01 100644 --- a/airbyte-integrations/connectors/source-recurly/setup.py +++ b/airbyte-integrations/connectors/source-recurly/setup.py @@ -14,6 +14,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-recurly=source_recurly.run:run", + ], + }, name="source_recurly", description="Source implementation for Recurly.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/run.py b/airbyte-integrations/connectors/source-recurly/source_recurly/run.py new file mode 100644 index 000000000000..746b6556605c --- /dev/null +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_recurly import SourceRecurly + + +def run(): + source = SourceRecurly() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-reply-io/main.py b/airbyte-integrations/connectors/source-reply-io/main.py index 879d184cebcd..ebe3ff2bfe7b 100644 --- a/airbyte-integrations/connectors/source-reply-io/main.py +++ b/airbyte-integrations/connectors/source-reply-io/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_reply_io import SourceReplyIo +from source_reply_io.run import run if __name__ == "__main__": - source = SourceReplyIo() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-reply-io/setup.py b/airbyte-integrations/connectors/source-reply-io/setup.py index bc4d841b50fc..4b8ad096a754 100644 --- a/airbyte-integrations/connectors/source-reply-io/setup.py +++ b/airbyte-integrations/connectors/source-reply-io/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-reply-io=source_reply_io.run:run", + ], + }, name="source_reply_io", description="Source implementation for Reply Io.", author="Elliot Trabac", author_email="elliot.trabac1@gmail.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-reply-io/source_reply_io/run.py b/airbyte-integrations/connectors/source-reply-io/source_reply_io/run.py new file mode 100644 index 000000000000..e740b708c45a --- /dev/null +++ b/airbyte-integrations/connectors/source-reply-io/source_reply_io/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_reply_io import SourceReplyIo + + +def run(): + source = SourceReplyIo() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-retently/main.py b/airbyte-integrations/connectors/source-retently/main.py index 4aafda0c119f..dec9d2267e7b 100644 --- a/airbyte-integrations/connectors/source-retently/main.py +++ b/airbyte-integrations/connectors/source-retently/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_retently import SourceRetently +from source_retently.run import run if __name__ == "__main__": - source = SourceRetently() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-retently/setup.py b/airbyte-integrations/connectors/source-retently/setup.py index 707ffcee736f..daf9a0a93efc 100644 --- a/airbyte-integrations/connectors/source-retently/setup.py +++ b/airbyte-integrations/connectors/source-retently/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-retently=source_retently.run:run", + ], + }, name="source_retently", description="Source implementation for Retently.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-retently/source_retently/run.py b/airbyte-integrations/connectors/source-retently/source_retently/run.py new file mode 100644 index 000000000000..4e0687b5c6a9 --- /dev/null +++ b/airbyte-integrations/connectors/source-retently/source_retently/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_retently import SourceRetently + + +def run(): + source = SourceRetently() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-ringcentral/main.py b/airbyte-integrations/connectors/source-ringcentral/main.py index cb412fcab28f..12f2eb4285cc 100644 --- a/airbyte-integrations/connectors/source-ringcentral/main.py +++ b/airbyte-integrations/connectors/source-ringcentral/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_ringcentral import SourceRingcentral +from source_ringcentral.run import run if __name__ == "__main__": - source = SourceRingcentral() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-ringcentral/setup.py b/airbyte-integrations/connectors/source-ringcentral/setup.py index a0a34d080b35..90d664f5d9f6 100644 --- a/airbyte-integrations/connectors/source-ringcentral/setup.py +++ b/airbyte-integrations/connectors/source-ringcentral/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-ringcentral=source_ringcentral.run:run", + ], + }, name="source_ringcentral", description="Source implementation for Ringcentral.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-ringcentral/source_ringcentral/run.py b/airbyte-integrations/connectors/source-ringcentral/source_ringcentral/run.py new file mode 100644 index 000000000000..f203a14ce2d1 --- /dev/null +++ b/airbyte-integrations/connectors/source-ringcentral/source_ringcentral/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_ringcentral import SourceRingcentral + + +def run(): + source = SourceRingcentral() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-rki-covid/main.py b/airbyte-integrations/connectors/source-rki-covid/main.py index 7b345c79d6d5..a104106001cb 100644 --- a/airbyte-integrations/connectors/source-rki-covid/main.py +++ b/airbyte-integrations/connectors/source-rki-covid/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_rki_covid import SourceRkiCovid +from source_rki_covid.run import run if __name__ == "__main__": - source = SourceRkiCovid() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-rki-covid/setup.py b/airbyte-integrations/connectors/source-rki-covid/setup.py index 4c8abfd77674..78722548d0b1 100644 --- a/airbyte-integrations/connectors/source-rki-covid/setup.py +++ b/airbyte-integrations/connectors/source-rki-covid/setup.py @@ -12,6 +12,11 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", "airbyte-cdk"] setup( + entry_points={ + "console_scripts": [ + "source-rki-covid=source_rki_covid.run:run", + ], + }, name="source_rki_covid", description="Source implementation for Rki Covid.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/run.py b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/run.py new file mode 100644 index 000000000000..dc5cdd95d152 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_rki_covid import SourceRkiCovid + + +def run(): + source = SourceRkiCovid() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-rocket-chat/main.py b/airbyte-integrations/connectors/source-rocket-chat/main.py index befcababa0fe..3e9e4cdfb68e 100644 --- a/airbyte-integrations/connectors/source-rocket-chat/main.py +++ b/airbyte-integrations/connectors/source-rocket-chat/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_rocket_chat import SourceRocketChat +from source_rocket_chat.run import run if __name__ == "__main__": - source = SourceRocketChat() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-rocket-chat/setup.py b/airbyte-integrations/connectors/source-rocket-chat/setup.py index 86e7d41371ab..0dea0d1fba8f 100644 --- a/airbyte-integrations/connectors/source-rocket-chat/setup.py +++ b/airbyte-integrations/connectors/source-rocket-chat/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-rocket-chat=source_rocket_chat.run:run", + ], + }, name="source_rocket_chat", description="Source implementation for Rocket Chat.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-rocket-chat/source_rocket_chat/run.py b/airbyte-integrations/connectors/source-rocket-chat/source_rocket_chat/run.py new file mode 100644 index 000000000000..1cb56e281978 --- /dev/null +++ b/airbyte-integrations/connectors/source-rocket-chat/source_rocket_chat/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_rocket_chat import SourceRocketChat + + +def run(): + source = SourceRocketChat() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-rss/main.py b/airbyte-integrations/connectors/source-rss/main.py index 967135ca3254..9e21c3c97793 100644 --- a/airbyte-integrations/connectors/source-rss/main.py +++ b/airbyte-integrations/connectors/source-rss/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_rss import SourceRss +from source_rss.run import run if __name__ == "__main__": - source = SourceRss() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-rss/setup.py b/airbyte-integrations/connectors/source-rss/setup.py index 7bd87cde621b..f49ef214cb6a 100644 --- a/airbyte-integrations/connectors/source-rss/setup.py +++ b/airbyte-integrations/connectors/source-rss/setup.py @@ -14,13 +14,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-rss=source_rss.run:run", + ], + }, name="source_rss", description="Source implementation for Rss.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-rss/source_rss/run.py b/airbyte-integrations/connectors/source-rss/source_rss/run.py new file mode 100644 index 000000000000..90f8a101fcfa --- /dev/null +++ b/airbyte-integrations/connectors/source-rss/source_rss/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_rss import SourceRss + + +def run(): + source = SourceRss() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-salesforce/setup.py b/airbyte-integrations/connectors/source-salesforce/setup.py index 4add132d7cb5..2e7fc4fff78d 100644 --- a/airbyte-integrations/connectors/source-salesforce/setup.py +++ b/airbyte-integrations/connectors/source-salesforce/setup.py @@ -16,7 +16,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-salesloft/main.py b/airbyte-integrations/connectors/source-salesloft/main.py index 8ee79923b913..6ae4308e9ab6 100644 --- a/airbyte-integrations/connectors/source-salesloft/main.py +++ b/airbyte-integrations/connectors/source-salesloft/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_salesloft import SourceSalesloft +from source_salesloft.run import run if __name__ == "__main__": - source = SourceSalesloft() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-salesloft/setup.py b/airbyte-integrations/connectors/source-salesloft/setup.py index bd9fc0a48179..58bcf6176dd6 100644 --- a/airbyte-integrations/connectors/source-salesloft/setup.py +++ b/airbyte-integrations/connectors/source-salesloft/setup.py @@ -14,6 +14,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-salesloft=source_salesloft.run:run", + ], + }, name="source_salesloft", description="Source implementation for Salesloft.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-salesloft/source_salesloft/run.py b/airbyte-integrations/connectors/source-salesloft/source_salesloft/run.py new file mode 100644 index 000000000000..e68c3785893c --- /dev/null +++ b/airbyte-integrations/connectors/source-salesloft/source_salesloft/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_salesloft import SourceSalesloft + + +def run(): + source = SourceSalesloft() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-sap-fieldglass/main.py b/airbyte-integrations/connectors/source-sap-fieldglass/main.py index c9273ef8ae47..8d8230917289 100644 --- a/airbyte-integrations/connectors/source-sap-fieldglass/main.py +++ b/airbyte-integrations/connectors/source-sap-fieldglass/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_sap_fieldglass import SourceSapFieldglass +from source_sap_fieldglass.run import run if __name__ == "__main__": - source = SourceSapFieldglass() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-sap-fieldglass/setup.py b/airbyte-integrations/connectors/source-sap-fieldglass/setup.py index 0f5e02083b5b..7365a4beb5ce 100644 --- a/airbyte-integrations/connectors/source-sap-fieldglass/setup.py +++ b/airbyte-integrations/connectors/source-sap-fieldglass/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-sap-fieldglass=source_sap_fieldglass.run:run", + ], + }, name="source_sap_fieldglass", description="Source implementation for Sap Fieldglass.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-sap-fieldglass/source_sap_fieldglass/run.py b/airbyte-integrations/connectors/source-sap-fieldglass/source_sap_fieldglass/run.py new file mode 100644 index 000000000000..69768a3d9efc --- /dev/null +++ b/airbyte-integrations/connectors/source-sap-fieldglass/source_sap_fieldglass/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_sap_fieldglass import SourceSapFieldglass + + +def run(): + source = SourceSapFieldglass() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/main.py b/airbyte-integrations/connectors/source-scaffold-source-http/main.py index f8030f72e927..f51b0283a681 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/main.py +++ b/airbyte-integrations/connectors/source-scaffold-source-http/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_scaffold_source_http import SourceScaffoldSourceHttp +from source_scaffold_source_http.run import run if __name__ == "__main__": - source = SourceScaffoldSourceHttp() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/setup.py b/airbyte-integrations/connectors/source-scaffold-source-http/setup.py index 4fbb5fc38b4d..a7a496b52161 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/setup.py +++ b/airbyte-integrations/connectors/source-scaffold-source-http/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-scaffold-source-http=source_scaffold_source_http.run:run", + ], + }, name="source_scaffold_source_http", description="Source implementation for Scaffold Source Http.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/source_scaffold_source_http/run.py b/airbyte-integrations/connectors/source-scaffold-source-http/source_scaffold_source_http/run.py new file mode 100644 index 000000000000..94b4f015f312 --- /dev/null +++ b/airbyte-integrations/connectors/source-scaffold-source-http/source_scaffold_source_http/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_scaffold_source_http import SourceScaffoldSourceHttp + + +def run(): + source = SourceScaffoldSourceHttp() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/main.py b/airbyte-integrations/connectors/source-scaffold-source-python/main.py index 0b8a20830163..038eaf6dba55 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/main.py +++ b/airbyte-integrations/connectors/source-scaffold-source-python/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_scaffold_source_python import SourceScaffoldSourcePython +from source_scaffold_source_python.run import run if __name__ == "__main__": - source = SourceScaffoldSourcePython() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/setup.py b/airbyte-integrations/connectors/source-scaffold-source-python/setup.py index b302f081011f..d4b3ebe7601b 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/setup.py +++ b/airbyte-integrations/connectors/source-scaffold-source-python/setup.py @@ -17,6 +17,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-scaffold-source-python=source_scaffold_source_python.run:run", + ], + }, name="source_scaffold_source_python", description="Source implementation for Scaffold Source Python.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/source_scaffold_source_python/run.py b/airbyte-integrations/connectors/source-scaffold-source-python/source_scaffold_source_python/run.py new file mode 100644 index 000000000000..6bb55fc68e7d --- /dev/null +++ b/airbyte-integrations/connectors/source-scaffold-source-python/source_scaffold_source_python/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_scaffold_source_python import SourceScaffoldSourcePython + + +def run(): + source = SourceScaffoldSourcePython() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-search-metrics/main.py b/airbyte-integrations/connectors/source-search-metrics/main.py index c07a9cf2a41b..29e5e8a133a6 100644 --- a/airbyte-integrations/connectors/source-search-metrics/main.py +++ b/airbyte-integrations/connectors/source-search-metrics/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_search_metrics import SourceSearchMetrics +from source_search_metrics.run import run if __name__ == "__main__": - source = SourceSearchMetrics() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-search-metrics/setup.py b/airbyte-integrations/connectors/source-search-metrics/setup.py index fda5333cc2ae..dfc33d408c22 100644 --- a/airbyte-integrations/connectors/source-search-metrics/setup.py +++ b/airbyte-integrations/connectors/source-search-metrics/setup.py @@ -16,6 +16,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-search-metrics=source_search_metrics.run:run", + ], + }, name="source_search_metrics", description="Source implementation for Search Metrics.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-search-metrics/source_search_metrics/run.py b/airbyte-integrations/connectors/source-search-metrics/source_search_metrics/run.py new file mode 100644 index 000000000000..29a0ebc75add --- /dev/null +++ b/airbyte-integrations/connectors/source-search-metrics/source_search_metrics/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_search_metrics import SourceSearchMetrics + + +def run(): + source = SourceSearchMetrics() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-secoda/main.py b/airbyte-integrations/connectors/source-secoda/main.py index 81d52e2f8a48..96bc9de24afa 100644 --- a/airbyte-integrations/connectors/source-secoda/main.py +++ b/airbyte-integrations/connectors/source-secoda/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_secoda import SourceSecoda +from source_secoda.run import run if __name__ == "__main__": - source = SourceSecoda() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-secoda/setup.py b/airbyte-integrations/connectors/source-secoda/setup.py index 3b603c1d6e21..b80c65173e94 100644 --- a/airbyte-integrations/connectors/source-secoda/setup.py +++ b/airbyte-integrations/connectors/source-secoda/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-secoda=source_secoda.run:run", + ], + }, name="source_secoda", description="Source implementation for Secoda.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-secoda/source_secoda/run.py b/airbyte-integrations/connectors/source-secoda/source_secoda/run.py new file mode 100644 index 000000000000..ed456d531ce1 --- /dev/null +++ b/airbyte-integrations/connectors/source-secoda/source_secoda/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_secoda import SourceSecoda + + +def run(): + source = SourceSecoda() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-sendgrid/main.py b/airbyte-integrations/connectors/source-sendgrid/main.py index f9720b12f1c8..e1c5a04db60d 100644 --- a/airbyte-integrations/connectors/source-sendgrid/main.py +++ b/airbyte-integrations/connectors/source-sendgrid/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_sendgrid import SourceSendgrid +from source_sendgrid.run import run if __name__ == "__main__": - source = SourceSendgrid() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-sendgrid/setup.py b/airbyte-integrations/connectors/source-sendgrid/setup.py index 64dc92cce236..0bfa4f9dd3b2 100644 --- a/airbyte-integrations/connectors/source-sendgrid/setup.py +++ b/airbyte-integrations/connectors/source-sendgrid/setup.py @@ -14,6 +14,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-sendgrid=source_sendgrid.run:run", + ], + }, name="source_sendgrid", description="Source implementation for Sendgrid.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/run.py b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/run.py new file mode 100644 index 000000000000..d2c1b798edb9 --- /dev/null +++ b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_sendgrid import SourceSendgrid + + +def run(): + source = SourceSendgrid() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-sendinblue/main.py b/airbyte-integrations/connectors/source-sendinblue/main.py index 8f9399e259f2..7227005ddf6f 100644 --- a/airbyte-integrations/connectors/source-sendinblue/main.py +++ b/airbyte-integrations/connectors/source-sendinblue/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_sendinblue import SourceSendinblue +from source_sendinblue.run import run if __name__ == "__main__": - source = SourceSendinblue() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-sendinblue/setup.py b/airbyte-integrations/connectors/source-sendinblue/setup.py index 0b5269c46b84..e452b66f447a 100644 --- a/airbyte-integrations/connectors/source-sendinblue/setup.py +++ b/airbyte-integrations/connectors/source-sendinblue/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-sendinblue=source_sendinblue.run:run", + ], + }, name="source_sendinblue", description="Source implementation for Sendinblue.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/run.py b/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/run.py new file mode 100644 index 000000000000..fbf77ce5a97a --- /dev/null +++ b/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_sendinblue import SourceSendinblue + + +def run(): + source = SourceSendinblue() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-senseforce/main.py b/airbyte-integrations/connectors/source-senseforce/main.py index ba269b9ba0ef..3a6a5e7fd4c7 100644 --- a/airbyte-integrations/connectors/source-senseforce/main.py +++ b/airbyte-integrations/connectors/source-senseforce/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_senseforce import SourceSenseforce +from source_senseforce.run import run if __name__ == "__main__": - source = SourceSenseforce() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-senseforce/setup.py b/airbyte-integrations/connectors/source-senseforce/setup.py index 976590623a83..4c653a13c824 100644 --- a/airbyte-integrations/connectors/source-senseforce/setup.py +++ b/airbyte-integrations/connectors/source-senseforce/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-senseforce=source_senseforce.run:run", + ], + }, name="source_senseforce", description="Source implementation for Senseforce.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-senseforce/source_senseforce/run.py b/airbyte-integrations/connectors/source-senseforce/source_senseforce/run.py new file mode 100644 index 000000000000..944e3468fed7 --- /dev/null +++ b/airbyte-integrations/connectors/source-senseforce/source_senseforce/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_senseforce import SourceSenseforce + + +def run(): + source = SourceSenseforce() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-sentry/main.py b/airbyte-integrations/connectors/source-sentry/main.py index 0844bc450390..1c7adc746e97 100644 --- a/airbyte-integrations/connectors/source-sentry/main.py +++ b/airbyte-integrations/connectors/source-sentry/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_sentry import SourceSentry +from source_sentry.run import run if __name__ == "__main__": - source = SourceSentry() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-sentry/setup.py b/airbyte-integrations/connectors/source-sentry/setup.py index 18d172dcd942..d1677c524a4d 100644 --- a/airbyte-integrations/connectors/source-sentry/setup.py +++ b/airbyte-integrations/connectors/source-sentry/setup.py @@ -17,6 +17,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-sentry=source_sentry.run:run", + ], + }, name="source_sentry", description="Source implementation for Sentry.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/run.py b/airbyte-integrations/connectors/source-sentry/source_sentry/run.py new file mode 100644 index 000000000000..acf82b00a03d --- /dev/null +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_sentry import SourceSentry + + +def run(): + source = SourceSentry() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-serpstat/main.py b/airbyte-integrations/connectors/source-serpstat/main.py index 92fb7edc0474..75c87dbf86f9 100644 --- a/airbyte-integrations/connectors/source-serpstat/main.py +++ b/airbyte-integrations/connectors/source-serpstat/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_serpstat import SourceSerpstat +from source_serpstat.run import run if __name__ == "__main__": - source = SourceSerpstat() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-serpstat/setup.py b/airbyte-integrations/connectors/source-serpstat/setup.py index 42ab32a171b0..9c7f53024268 100644 --- a/airbyte-integrations/connectors/source-serpstat/setup.py +++ b/airbyte-integrations/connectors/source-serpstat/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-serpstat=source_serpstat.run:run", + ], + }, name="source_serpstat", description="Source implementation for Serpstat.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-serpstat/source_serpstat/run.py b/airbyte-integrations/connectors/source-serpstat/source_serpstat/run.py new file mode 100644 index 000000000000..3dc6051ee8f8 --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/source_serpstat/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_serpstat import SourceSerpstat + + +def run(): + source = SourceSerpstat() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-sftp-bulk/main.py b/airbyte-integrations/connectors/source-sftp-bulk/main.py index c2b1e7afb5b3..9e6488da04cc 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/main.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_sftp_bulk import SourceFtp +from source_sftp_bulk.run import run if __name__ == "__main__": - source = SourceFtp() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-sftp-bulk/setup.py b/airbyte-integrations/connectors/source-sftp-bulk/setup.py index 6d9d1990f634..e09d478f9007 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/setup.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/setup.py @@ -16,6 +16,11 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1", "docker==5.0.3"] setup( + entry_points={ + "console_scripts": [ + "source-sftp-bulk=source_sftp_bulk.run:run", + ], + }, name="source_sftp_bulk", description="Source implementation for SFTP Bulk.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/run.py b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/run.py new file mode 100644 index 000000000000..c3e00b8100cd --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_sftp_bulk import SourceFtp + + +def run(): + source = SourceFtp() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-shortio/main.py b/airbyte-integrations/connectors/source-shortio/main.py index 0c6be2ce4469..eaef1e24f90d 100644 --- a/airbyte-integrations/connectors/source-shortio/main.py +++ b/airbyte-integrations/connectors/source-shortio/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_shortio import SourceShortio +from source_shortio.run import run if __name__ == "__main__": - source = SourceShortio() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-shortio/setup.py b/airbyte-integrations/connectors/source-shortio/setup.py index 6c8cbba1b880..d0b2d6cc581c 100644 --- a/airbyte-integrations/connectors/source-shortio/setup.py +++ b/airbyte-integrations/connectors/source-shortio/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-shortio=source_shortio.run:run", + ], + }, name="source_shortio", description="Source implementation for Shortio.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-shortio/source_shortio/run.py b/airbyte-integrations/connectors/source-shortio/source_shortio/run.py new file mode 100644 index 000000000000..dce618a444e4 --- /dev/null +++ b/airbyte-integrations/connectors/source-shortio/source_shortio/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_shortio import SourceShortio + + +def run(): + source = SourceShortio() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-smaily/main.py b/airbyte-integrations/connectors/source-smaily/main.py index 647f5a347f70..fbb55aa40a54 100644 --- a/airbyte-integrations/connectors/source-smaily/main.py +++ b/airbyte-integrations/connectors/source-smaily/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_smaily import SourceSmaily +from source_smaily.run import run if __name__ == "__main__": - source = SourceSmaily() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-smaily/setup.py b/airbyte-integrations/connectors/source-smaily/setup.py index aa835b3698f6..f3b23a618c6a 100644 --- a/airbyte-integrations/connectors/source-smaily/setup.py +++ b/airbyte-integrations/connectors/source-smaily/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-smaily=source_smaily.run:run", + ], + }, name="source_smaily", description="Source implementation for Smaily.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/run.py b/airbyte-integrations/connectors/source-smaily/source_smaily/run.py new file mode 100644 index 000000000000..24297e8addb3 --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_smaily import SourceSmaily + + +def run(): + source = SourceSmaily() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-smartengage/main.py b/airbyte-integrations/connectors/source-smartengage/main.py index f9aec77e67ec..d445802b0ed4 100644 --- a/airbyte-integrations/connectors/source-smartengage/main.py +++ b/airbyte-integrations/connectors/source-smartengage/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_smartengage import SourceSmartengage +from source_smartengage.run import run if __name__ == "__main__": - source = SourceSmartengage() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-smartengage/setup.py b/airbyte-integrations/connectors/source-smartengage/setup.py index 958a3b00df73..e4dd81fe7a9b 100644 --- a/airbyte-integrations/connectors/source-smartengage/setup.py +++ b/airbyte-integrations/connectors/source-smartengage/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-smartengage=source_smartengage.run:run", + ], + }, name="source_smartengage", description="Source implementation for Smartengage.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-smartengage/source_smartengage/run.py b/airbyte-integrations/connectors/source-smartengage/source_smartengage/run.py new file mode 100644 index 000000000000..593e4eaf40f1 --- /dev/null +++ b/airbyte-integrations/connectors/source-smartengage/source_smartengage/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_smartengage import SourceSmartengage + + +def run(): + source = SourceSmartengage() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/main.py b/airbyte-integrations/connectors/source-snapchat-marketing/main.py index 3b7dadc93a3e..d48c013d7dcf 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/main.py +++ b/airbyte-integrations/connectors/source-snapchat-marketing/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_snapchat_marketing import SourceSnapchatMarketing +from source_snapchat_marketing.run import run if __name__ == "__main__": - source = SourceSnapchatMarketing() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/setup.py b/airbyte-integrations/connectors/source-snapchat-marketing/setup.py index cc60f43cbe70..71d4210cbe54 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/setup.py +++ b/airbyte-integrations/connectors/source-snapchat-marketing/setup.py @@ -12,6 +12,11 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1", "requests_mock"] setup( + entry_points={ + "console_scripts": [ + "source-snapchat-marketing=source_snapchat_marketing.run:run", + ], + }, name="source_snapchat_marketing", description="Source implementation for Snapchat Marketing.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/run.py b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/run.py new file mode 100644 index 000000000000..61c972da8a2d --- /dev/null +++ b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_snapchat_marketing import SourceSnapchatMarketing + + +def run(): + source = SourceSnapchatMarketing() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-sonar-cloud/main.py b/airbyte-integrations/connectors/source-sonar-cloud/main.py index 81355fadfed1..5ba429ba3211 100644 --- a/airbyte-integrations/connectors/source-sonar-cloud/main.py +++ b/airbyte-integrations/connectors/source-sonar-cloud/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_sonar_cloud import SourceSonarCloud +from source_sonar_cloud.run import run if __name__ == "__main__": - source = SourceSonarCloud() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-sonar-cloud/setup.py b/airbyte-integrations/connectors/source-sonar-cloud/setup.py index e99684d61a65..2f8f82301482 100644 --- a/airbyte-integrations/connectors/source-sonar-cloud/setup.py +++ b/airbyte-integrations/connectors/source-sonar-cloud/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-sonar-cloud=source_sonar_cloud.run:run", + ], + }, name="source_sonar_cloud", description="Source implementation for Sonar Cloud.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/run.py b/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/run.py new file mode 100644 index 000000000000..e29261106741 --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_sonar_cloud import SourceSonarCloud + + +def run(): + source = SourceSonarCloud() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-spacex-api/main.py b/airbyte-integrations/connectors/source-spacex-api/main.py index 7ceab1979a8b..52f8d09f4508 100644 --- a/airbyte-integrations/connectors/source-spacex-api/main.py +++ b/airbyte-integrations/connectors/source-spacex-api/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_spacex_api import SourceSpacexApi +from source_spacex_api.run import run if __name__ == "__main__": - source = SourceSpacexApi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-spacex-api/setup.py b/airbyte-integrations/connectors/source-spacex-api/setup.py index 2d202b30ec9a..00b9cc0110f4 100644 --- a/airbyte-integrations/connectors/source-spacex-api/setup.py +++ b/airbyte-integrations/connectors/source-spacex-api/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-spacex-api=source_spacex_api.run:run", + ], + }, name="source_spacex_api", description="Source implementation for Spacex Api.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/run.py b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/run.py new file mode 100644 index 000000000000..7baf4913b453 --- /dev/null +++ b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_spacex_api import SourceSpacexApi + + +def run(): + source = SourceSpacexApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-square/main.py b/airbyte-integrations/connectors/source-square/main.py index 064f99648db8..7102d93900cf 100644 --- a/airbyte-integrations/connectors/source-square/main.py +++ b/airbyte-integrations/connectors/source-square/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_square import SourceSquare +from source_square.run import run if __name__ == "__main__": - source = SourceSquare() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-square/setup.py b/airbyte-integrations/connectors/source-square/setup.py index fe3012decd5d..3b7c2204ad12 100644 --- a/airbyte-integrations/connectors/source-square/setup.py +++ b/airbyte-integrations/connectors/source-square/setup.py @@ -17,13 +17,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-square=source_square.run:run", + ], + }, name="source_square", description="Source implementation for Square.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-square/source_square/run.py b/airbyte-integrations/connectors/source-square/source_square/run.py new file mode 100644 index 000000000000..df5f3fd74012 --- /dev/null +++ b/airbyte-integrations/connectors/source-square/source_square/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_square import SourceSquare + + +def run(): + source = SourceSquare() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-statuspage/main.py b/airbyte-integrations/connectors/source-statuspage/main.py index 8046f076e8ab..2701aa6de07d 100644 --- a/airbyte-integrations/connectors/source-statuspage/main.py +++ b/airbyte-integrations/connectors/source-statuspage/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_statuspage import SourceStatuspage +from source_statuspage.run import run if __name__ == "__main__": - source = SourceStatuspage() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-statuspage/setup.py b/airbyte-integrations/connectors/source-statuspage/setup.py index 772fedc5e034..f7adbd694ed6 100644 --- a/airbyte-integrations/connectors/source-statuspage/setup.py +++ b/airbyte-integrations/connectors/source-statuspage/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-statuspage=source_statuspage.run:run", + ], + }, name="source_statuspage", description="Source implementation for Statuspage.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-statuspage/source_statuspage/run.py b/airbyte-integrations/connectors/source-statuspage/source_statuspage/run.py new file mode 100644 index 000000000000..d29716d7c02f --- /dev/null +++ b/airbyte-integrations/connectors/source-statuspage/source_statuspage/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_statuspage import SourceStatuspage + + +def run(): + source = SourceStatuspage() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-strava/main.py b/airbyte-integrations/connectors/source-strava/main.py index 054e4911089c..9fc5a4bb2918 100644 --- a/airbyte-integrations/connectors/source-strava/main.py +++ b/airbyte-integrations/connectors/source-strava/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_strava import SourceStrava +from source_strava.run import run if __name__ == "__main__": - source = SourceStrava() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-strava/setup.py b/airbyte-integrations/connectors/source-strava/setup.py index 8dda89709cd4..3bdc92646070 100644 --- a/airbyte-integrations/connectors/source-strava/setup.py +++ b/airbyte-integrations/connectors/source-strava/setup.py @@ -14,13 +14,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-strava=source_strava.run:run", + ], + }, name="source_strava", description="Source implementation for Strava.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-strava/source_strava/run.py b/airbyte-integrations/connectors/source-strava/source_strava/run.py new file mode 100644 index 000000000000..c8b1590993cd --- /dev/null +++ b/airbyte-integrations/connectors/source-strava/source_strava/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_strava import SourceStrava + + +def run(): + source = SourceStrava() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-stripe/setup.py b/airbyte-integrations/connectors/source-stripe/setup.py index c1b020664edb..e0203e616ad7 100644 --- a/airbyte-integrations/connectors/source-stripe/setup.py +++ b/airbyte-integrations/connectors/source-stripe/setup.py @@ -17,7 +17,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-survey-sparrow/main.py b/airbyte-integrations/connectors/source-survey-sparrow/main.py index 31056359d13b..5c4977542eaf 100644 --- a/airbyte-integrations/connectors/source-survey-sparrow/main.py +++ b/airbyte-integrations/connectors/source-survey-sparrow/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_survey_sparrow import SourceSurveySparrow +from source_survey_sparrow.run import run if __name__ == "__main__": - source = SourceSurveySparrow() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-survey-sparrow/setup.py b/airbyte-integrations/connectors/source-survey-sparrow/setup.py index da1ca261a46e..dfa0601abbde 100644 --- a/airbyte-integrations/connectors/source-survey-sparrow/setup.py +++ b/airbyte-integrations/connectors/source-survey-sparrow/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-survey-sparrow=source_survey_sparrow.run:run", + ], + }, name="source_survey_sparrow", description="Source implementation for Survey Sparrow.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-survey-sparrow/source_survey_sparrow/run.py b/airbyte-integrations/connectors/source-survey-sparrow/source_survey_sparrow/run.py new file mode 100644 index 000000000000..af39a841b3dd --- /dev/null +++ b/airbyte-integrations/connectors/source-survey-sparrow/source_survey_sparrow/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_survey_sparrow import SourceSurveySparrow + + +def run(): + source = SourceSurveySparrow() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-surveycto/main.py b/airbyte-integrations/connectors/source-surveycto/main.py index 4f26fe81785f..9f282dbc2ecd 100644 --- a/airbyte-integrations/connectors/source-surveycto/main.py +++ b/airbyte-integrations/connectors/source-surveycto/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_surveycto import SourceSurveycto +from source_surveycto.run import run if __name__ == "__main__": - source = SourceSurveycto() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-surveycto/setup.py b/airbyte-integrations/connectors/source-surveycto/setup.py index be5f78fbdd14..92e965e0651b 100644 --- a/airbyte-integrations/connectors/source-surveycto/setup.py +++ b/airbyte-integrations/connectors/source-surveycto/setup.py @@ -14,13 +14,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-surveycto=source_surveycto.run:run", + ], + }, name="source_surveycto", description="Source implementation for Surveycto.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-surveycto/source_surveycto/run.py b/airbyte-integrations/connectors/source-surveycto/source_surveycto/run.py new file mode 100644 index 000000000000..927d17c9eb1f --- /dev/null +++ b/airbyte-integrations/connectors/source-surveycto/source_surveycto/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_surveycto import SourceSurveycto + + +def run(): + source = SourceSurveycto() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-surveymonkey/main.py b/airbyte-integrations/connectors/source-surveymonkey/main.py index 7fd72e7dd38f..bf4f900ad377 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/main.py +++ b/airbyte-integrations/connectors/source-surveymonkey/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_surveymonkey import SourceSurveymonkey +from source_surveymonkey.run import run if __name__ == "__main__": - source = SourceSurveymonkey() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-surveymonkey/setup.py b/airbyte-integrations/connectors/source-surveymonkey/setup.py index f70f3e894857..a72a0108a10b 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/setup.py +++ b/airbyte-integrations/connectors/source-surveymonkey/setup.py @@ -10,6 +10,11 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1", "requests_mock"] setup( + entry_points={ + "console_scripts": [ + "source-surveymonkey=source_surveymonkey.run:run", + ], + }, name="source_surveymonkey", description="Source implementation for Surveymonkey.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/run.py b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/run.py new file mode 100644 index 000000000000..f3cbc028402b --- /dev/null +++ b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_surveymonkey import SourceSurveymonkey + + +def run(): + source = SourceSurveymonkey() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-talkdesk-explore/main.py b/airbyte-integrations/connectors/source-talkdesk-explore/main.py index 33dcea10e90a..745a3f67e001 100644 --- a/airbyte-integrations/connectors/source-talkdesk-explore/main.py +++ b/airbyte-integrations/connectors/source-talkdesk-explore/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_talkdesk_explore import SourceTalkdeskExplore +from source_talkdesk_explore.run import run if __name__ == "__main__": - source = SourceTalkdeskExplore() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-talkdesk-explore/setup.py b/airbyte-integrations/connectors/source-talkdesk-explore/setup.py index 2694e175a333..5df2ba2853d0 100644 --- a/airbyte-integrations/connectors/source-talkdesk-explore/setup.py +++ b/airbyte-integrations/connectors/source-talkdesk-explore/setup.py @@ -16,6 +16,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-talkdesk-explore=source_talkdesk_explore.run:run", + ], + }, name="source_talkdesk_explore", description="Source implementation for Talkdesk Explore API.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-talkdesk-explore/source_talkdesk_explore/run.py b/airbyte-integrations/connectors/source-talkdesk-explore/source_talkdesk_explore/run.py new file mode 100644 index 000000000000..442b84e5a278 --- /dev/null +++ b/airbyte-integrations/connectors/source-talkdesk-explore/source_talkdesk_explore/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_talkdesk_explore import SourceTalkdeskExplore + + +def run(): + source = SourceTalkdeskExplore() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-tempo/main.py b/airbyte-integrations/connectors/source-tempo/main.py index e26fc47cd806..d8f0e748fb2f 100644 --- a/airbyte-integrations/connectors/source-tempo/main.py +++ b/airbyte-integrations/connectors/source-tempo/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_tempo import SourceTempo +from source_tempo.run import run if __name__ == "__main__": - source = SourceTempo() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-tempo/setup.py b/airbyte-integrations/connectors/source-tempo/setup.py index 16051a8dfa6b..3e2668d10d6c 100644 --- a/airbyte-integrations/connectors/source-tempo/setup.py +++ b/airbyte-integrations/connectors/source-tempo/setup.py @@ -16,6 +16,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-tempo=source_tempo.run:run", + ], + }, name="source_tempo", description="Source implementation for Tempo.", author="Thomas van Latum", diff --git a/airbyte-integrations/connectors/source-tempo/source_tempo/run.py b/airbyte-integrations/connectors/source-tempo/source_tempo/run.py new file mode 100644 index 000000000000..8883fc3f0a24 --- /dev/null +++ b/airbyte-integrations/connectors/source-tempo/source_tempo/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_tempo import SourceTempo + + +def run(): + source = SourceTempo() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-the-guardian-api/main.py b/airbyte-integrations/connectors/source-the-guardian-api/main.py index 96a88ceac666..50182e0c2da0 100644 --- a/airbyte-integrations/connectors/source-the-guardian-api/main.py +++ b/airbyte-integrations/connectors/source-the-guardian-api/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_the_guardian_api import SourceTheGuardianApi +from source_the_guardian_api.run import run if __name__ == "__main__": - source = SourceTheGuardianApi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-the-guardian-api/setup.py b/airbyte-integrations/connectors/source-the-guardian-api/setup.py index 1fb11d947926..9627db8db884 100644 --- a/airbyte-integrations/connectors/source-the-guardian-api/setup.py +++ b/airbyte-integrations/connectors/source-the-guardian-api/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-the-guardian-api=source_the_guardian_api.run:run", + ], + }, name="source_the_guardian_api", description="Source implementation for The Guardian Api.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-the-guardian-api/source_the_guardian_api/run.py b/airbyte-integrations/connectors/source-the-guardian-api/source_the_guardian_api/run.py new file mode 100644 index 000000000000..eafbdfd35301 --- /dev/null +++ b/airbyte-integrations/connectors/source-the-guardian-api/source_the_guardian_api/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_the_guardian_api import SourceTheGuardianApi + + +def run(): + source = SourceTheGuardianApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/main.py b/airbyte-integrations/connectors/source-tiktok-marketing/main.py index d20d9e211aa8..b523ea1b0fdd 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/main.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_tiktok_marketing import SourceTiktokMarketing +from source_tiktok_marketing.run import run if __name__ == "__main__": - source = SourceTiktokMarketing() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/setup.py b/airbyte-integrations/connectors/source-tiktok-marketing/setup.py index 3a5d282355a8..21896188c6be 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/setup.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/setup.py @@ -10,6 +10,11 @@ TEST_REQUIREMENTS = ["pytest-mock~=3.6.1", "pytest~=6.1", "requests-mock==1.9.3", "timeout-decorator==0.5.0"] setup( + entry_points={ + "console_scripts": [ + "source-tiktok-marketing=source_tiktok_marketing.run:run", + ], + }, name="source_tiktok_marketing", description="Source implementation for Tiktok Marketing.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/run.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/run.py new file mode 100644 index 000000000000..341638b66529 --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_tiktok_marketing import SourceTiktokMarketing + + +def run(): + source = SourceTiktokMarketing() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-timely/main.py b/airbyte-integrations/connectors/source-timely/main.py index 674590cf1d99..1007e1945bc0 100644 --- a/airbyte-integrations/connectors/source-timely/main.py +++ b/airbyte-integrations/connectors/source-timely/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_timely import SourceTimely +from source_timely.run import run if __name__ == "__main__": - source = SourceTimely() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-timely/setup.py b/airbyte-integrations/connectors/source-timely/setup.py index 6dc55b4722ef..9004e784b613 100644 --- a/airbyte-integrations/connectors/source-timely/setup.py +++ b/airbyte-integrations/connectors/source-timely/setup.py @@ -10,13 +10,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-timely=source_timely.run:run", + ], + }, name="source_timely", description="Source implementation for Timely.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-timely/source_timely/run.py b/airbyte-integrations/connectors/source-timely/source_timely/run.py new file mode 100644 index 000000000000..6ead8b999eb9 --- /dev/null +++ b/airbyte-integrations/connectors/source-timely/source_timely/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_timely import SourceTimely + + +def run(): + source = SourceTimely() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-tmdb/main.py b/airbyte-integrations/connectors/source-tmdb/main.py index 8fb1b3a13296..de1e36196011 100644 --- a/airbyte-integrations/connectors/source-tmdb/main.py +++ b/airbyte-integrations/connectors/source-tmdb/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_tmdb import SourceTmdb +from source_tmdb.run import run if __name__ == "__main__": - source = SourceTmdb() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-tmdb/setup.py b/airbyte-integrations/connectors/source-tmdb/setup.py index 35310f227836..d1b20699bcf3 100644 --- a/airbyte-integrations/connectors/source-tmdb/setup.py +++ b/airbyte-integrations/connectors/source-tmdb/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-tmdb=source_tmdb.run:run", + ], + }, name="source_tmdb", description="Source implementation for Tmdb.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-tmdb/source_tmdb/run.py b/airbyte-integrations/connectors/source-tmdb/source_tmdb/run.py new file mode 100644 index 000000000000..100edabab0b1 --- /dev/null +++ b/airbyte-integrations/connectors/source-tmdb/source_tmdb/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_tmdb import SourceTmdb + + +def run(): + source = SourceTmdb() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-todoist/main.py b/airbyte-integrations/connectors/source-todoist/main.py index 21ae17e05de7..159e7dcdc036 100644 --- a/airbyte-integrations/connectors/source-todoist/main.py +++ b/airbyte-integrations/connectors/source-todoist/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_todoist import SourceTodoist +from source_todoist.run import run if __name__ == "__main__": - source = SourceTodoist() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-todoist/setup.py b/airbyte-integrations/connectors/source-todoist/setup.py index 3add92262189..601df38fd542 100644 --- a/airbyte-integrations/connectors/source-todoist/setup.py +++ b/airbyte-integrations/connectors/source-todoist/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-todoist=source_todoist.run:run", + ], + }, name="source_todoist", description="Source implementation for Todoist.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/run.py b/airbyte-integrations/connectors/source-todoist/source_todoist/run.py new file mode 100644 index 000000000000..b20e5a2d580f --- /dev/null +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_todoist import SourceTodoist + + +def run(): + source = SourceTodoist() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-toggl/main.py b/airbyte-integrations/connectors/source-toggl/main.py index bd1b4defb2de..a52cb2ff9370 100644 --- a/airbyte-integrations/connectors/source-toggl/main.py +++ b/airbyte-integrations/connectors/source-toggl/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_toggl import SourceToggl +from source_toggl.run import run if __name__ == "__main__": - source = SourceToggl() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-toggl/setup.py b/airbyte-integrations/connectors/source-toggl/setup.py index bb481cee5328..cdd1baad6ba7 100644 --- a/airbyte-integrations/connectors/source-toggl/setup.py +++ b/airbyte-integrations/connectors/source-toggl/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-toggl=source_toggl.run:run", + ], + }, name="source_toggl", description="Source implementation for Toggl.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-toggl/source_toggl/run.py b/airbyte-integrations/connectors/source-toggl/source_toggl/run.py new file mode 100644 index 000000000000..fd06c41abef3 --- /dev/null +++ b/airbyte-integrations/connectors/source-toggl/source_toggl/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_toggl import SourceToggl + + +def run(): + source = SourceToggl() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-tplcentral/main.py b/airbyte-integrations/connectors/source-tplcentral/main.py index e8e5f30138f4..c5e7b8a95ec4 100644 --- a/airbyte-integrations/connectors/source-tplcentral/main.py +++ b/airbyte-integrations/connectors/source-tplcentral/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_tplcentral import SourceTplcentral +from source_tplcentral.run import run if __name__ == "__main__": - source = SourceTplcentral() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-tplcentral/setup.py b/airbyte-integrations/connectors/source-tplcentral/setup.py index 602d12056cb3..45cd0ff8a633 100644 --- a/airbyte-integrations/connectors/source-tplcentral/setup.py +++ b/airbyte-integrations/connectors/source-tplcentral/setup.py @@ -17,6 +17,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-tplcentral=source_tplcentral.run:run", + ], + }, name="source_tplcentral", description="Source implementation for Tplcentral.", author="Labanoras Tech", diff --git a/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/run.py b/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/run.py new file mode 100644 index 000000000000..443a194ed6c1 --- /dev/null +++ b/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_tplcentral import SourceTplcentral + + +def run(): + source = SourceTplcentral() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-trello/main.py b/airbyte-integrations/connectors/source-trello/main.py index c0657c0065fc..7b0b502fdb1e 100644 --- a/airbyte-integrations/connectors/source-trello/main.py +++ b/airbyte-integrations/connectors/source-trello/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_trello import SourceTrello +from source_trello.run import run if __name__ == "__main__": - source = SourceTrello() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-trello/setup.py b/airbyte-integrations/connectors/source-trello/setup.py index 38d12ba64785..85fd6920b1af 100644 --- a/airbyte-integrations/connectors/source-trello/setup.py +++ b/airbyte-integrations/connectors/source-trello/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-trello=source_trello.run:run", + ], + }, name="source_trello", description="Source implementation for Trello.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-trello/source_trello/run.py b/airbyte-integrations/connectors/source-trello/source_trello/run.py new file mode 100644 index 000000000000..628d22faf0db --- /dev/null +++ b/airbyte-integrations/connectors/source-trello/source_trello/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_trello import SourceTrello + + +def run(): + source = SourceTrello() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-trustpilot/main.py b/airbyte-integrations/connectors/source-trustpilot/main.py index f0cd7b4265e5..27e409441e33 100644 --- a/airbyte-integrations/connectors/source-trustpilot/main.py +++ b/airbyte-integrations/connectors/source-trustpilot/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_trustpilot import SourceTrustpilot +from source_trustpilot.run import run if __name__ == "__main__": - source = SourceTrustpilot() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-trustpilot/setup.py b/airbyte-integrations/connectors/source-trustpilot/setup.py index 4b8001807ec1..7c61c708d96e 100644 --- a/airbyte-integrations/connectors/source-trustpilot/setup.py +++ b/airbyte-integrations/connectors/source-trustpilot/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-trustpilot=source_trustpilot.run:run", + ], + }, name="source_trustpilot", description="Source implementation for Trustpilot.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-trustpilot/source_trustpilot/run.py b/airbyte-integrations/connectors/source-trustpilot/source_trustpilot/run.py new file mode 100644 index 000000000000..40c488dbf9a7 --- /dev/null +++ b/airbyte-integrations/connectors/source-trustpilot/source_trustpilot/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_trustpilot import SourceTrustpilot + + +def run(): + source = SourceTrustpilot() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/main.py b/airbyte-integrations/connectors/source-tvmaze-schedule/main.py index 3a530f427a6f..be6abd8ef07c 100644 --- a/airbyte-integrations/connectors/source-tvmaze-schedule/main.py +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_tvmaze_schedule import SourceTvmazeSchedule +from source_tvmaze_schedule.run import run if __name__ == "__main__": - source = SourceTvmazeSchedule() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/setup.py b/airbyte-integrations/connectors/source-tvmaze-schedule/setup.py index 77c97057564e..800761e02c23 100644 --- a/airbyte-integrations/connectors/source-tvmaze-schedule/setup.py +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-tvmaze-schedule=source_tvmaze_schedule.run:run", + ], + }, name="source_tvmaze_schedule", description="Source implementation for Tvmaze Schedule.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/run.py b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/run.py new file mode 100644 index 000000000000..24cdb98bf699 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_tvmaze_schedule import SourceTvmazeSchedule + + +def run(): + source = SourceTvmazeSchedule() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-twilio-taskrouter/main.py b/airbyte-integrations/connectors/source-twilio-taskrouter/main.py index 792fa7db3284..94a3a9659fe9 100644 --- a/airbyte-integrations/connectors/source-twilio-taskrouter/main.py +++ b/airbyte-integrations/connectors/source-twilio-taskrouter/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_twilio_taskrouter import SourceTwilioTaskrouter +from source_twilio_taskrouter.run import run if __name__ == "__main__": - source = SourceTwilioTaskrouter() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-twilio-taskrouter/setup.py b/airbyte-integrations/connectors/source-twilio-taskrouter/setup.py index be86feef816f..2d7ea97c9833 100644 --- a/airbyte-integrations/connectors/source-twilio-taskrouter/setup.py +++ b/airbyte-integrations/connectors/source-twilio-taskrouter/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-twilio-taskrouter=source_twilio_taskrouter.run:run", + ], + }, name="source_twilio_taskrouter", description="Source implementation for Twilio Taskrouter.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-twilio-taskrouter/source_twilio_taskrouter/run.py b/airbyte-integrations/connectors/source-twilio-taskrouter/source_twilio_taskrouter/run.py new file mode 100644 index 000000000000..6c677af9dd2c --- /dev/null +++ b/airbyte-integrations/connectors/source-twilio-taskrouter/source_twilio_taskrouter/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_twilio_taskrouter import SourceTwilioTaskrouter + + +def run(): + source = SourceTwilioTaskrouter() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-twilio/main.py b/airbyte-integrations/connectors/source-twilio/main.py index 8f91bc04c56f..0999d1e67f26 100644 --- a/airbyte-integrations/connectors/source-twilio/main.py +++ b/airbyte-integrations/connectors/source-twilio/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_twilio import SourceTwilio +from source_twilio.run import run if __name__ == "__main__": - source = SourceTwilio() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-twilio/setup.py b/airbyte-integrations/connectors/source-twilio/setup.py index 66a2cb8a7ee0..19717cccc42f 100644 --- a/airbyte-integrations/connectors/source-twilio/setup.py +++ b/airbyte-integrations/connectors/source-twilio/setup.py @@ -14,6 +14,11 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock", "requests_mock", "freezegun"] setup( + entry_points={ + "console_scripts": [ + "source-twilio=source_twilio.run:run", + ], + }, name="source_twilio", description="Source implementation for Twilio.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/run.py b/airbyte-integrations/connectors/source-twilio/source_twilio/run.py new file mode 100644 index 000000000000..bb95086d7eba --- /dev/null +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_twilio import SourceTwilio + + +def run(): + source = SourceTwilio() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-twitter/main.py b/airbyte-integrations/connectors/source-twitter/main.py index 3e6dae60da34..f50219e6455c 100644 --- a/airbyte-integrations/connectors/source-twitter/main.py +++ b/airbyte-integrations/connectors/source-twitter/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_twitter import SourceTwitter +from source_twitter.run import run if __name__ == "__main__": - source = SourceTwitter() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-twitter/setup.py b/airbyte-integrations/connectors/source-twitter/setup.py index bcc921fcba46..e85c1d404a2a 100644 --- a/airbyte-integrations/connectors/source-twitter/setup.py +++ b/airbyte-integrations/connectors/source-twitter/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-twitter=source_twitter.run:run", + ], + }, name="source_twitter", description="Source implementation for Twitter.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-twitter/source_twitter/run.py b/airbyte-integrations/connectors/source-twitter/source_twitter/run.py new file mode 100644 index 000000000000..4c18e4636da4 --- /dev/null +++ b/airbyte-integrations/connectors/source-twitter/source_twitter/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_twitter import SourceTwitter + + +def run(): + source = SourceTwitter() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-tyntec-sms/main.py b/airbyte-integrations/connectors/source-tyntec-sms/main.py index 37e104247846..142de37101e6 100644 --- a/airbyte-integrations/connectors/source-tyntec-sms/main.py +++ b/airbyte-integrations/connectors/source-tyntec-sms/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_tyntec_sms import SourceTyntecSms +from source_tyntec_sms.run import run if __name__ == "__main__": - source = SourceTyntecSms() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-tyntec-sms/setup.py b/airbyte-integrations/connectors/source-tyntec-sms/setup.py index 1a15c6223694..832b4139ef2d 100644 --- a/airbyte-integrations/connectors/source-tyntec-sms/setup.py +++ b/airbyte-integrations/connectors/source-tyntec-sms/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-tyntec-sms=source_tyntec_sms.run:run", + ], + }, name="source_tyntec_sms", description="Source implementation for Tyntec Sms.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-tyntec-sms/source_tyntec_sms/run.py b/airbyte-integrations/connectors/source-tyntec-sms/source_tyntec_sms/run.py new file mode 100644 index 000000000000..cd08f8c0c7ee --- /dev/null +++ b/airbyte-integrations/connectors/source-tyntec-sms/source_tyntec_sms/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_tyntec_sms import SourceTyntecSms + + +def run(): + source = SourceTyntecSms() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-typeform/setup.py b/airbyte-integrations/connectors/source-typeform/setup.py index 09b119b93869..1caa440b9122 100644 --- a/airbyte-integrations/connectors/source-typeform/setup.py +++ b/airbyte-integrations/connectors/source-typeform/setup.py @@ -21,7 +21,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-unleash/main.py b/airbyte-integrations/connectors/source-unleash/main.py index 9e57236c979a..72ec2c888dcb 100644 --- a/airbyte-integrations/connectors/source-unleash/main.py +++ b/airbyte-integrations/connectors/source-unleash/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_unleash import SourceUnleash +from source_unleash.run import run if __name__ == "__main__": - source = SourceUnleash() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-unleash/setup.py b/airbyte-integrations/connectors/source-unleash/setup.py index 8c00746f3fd4..9822815f03ce 100644 --- a/airbyte-integrations/connectors/source-unleash/setup.py +++ b/airbyte-integrations/connectors/source-unleash/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-unleash=source_unleash.run:run", + ], + }, name="source_unleash", description="Source implementation for Unleash.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-unleash/source_unleash/run.py b/airbyte-integrations/connectors/source-unleash/source_unleash/run.py new file mode 100644 index 000000000000..0de14599b8b3 --- /dev/null +++ b/airbyte-integrations/connectors/source-unleash/source_unleash/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_unleash import SourceUnleash + + +def run(): + source = SourceUnleash() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-us-census/main.py b/airbyte-integrations/connectors/source-us-census/main.py index 95978af82418..c46fd28c9ce6 100644 --- a/airbyte-integrations/connectors/source-us-census/main.py +++ b/airbyte-integrations/connectors/source-us-census/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_us_census import SourceUsCensus +from source_us_census.run import run if __name__ == "__main__": - source = SourceUsCensus() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-us-census/setup.py b/airbyte-integrations/connectors/source-us-census/setup.py index 7b2b4ebc83ec..bf3858455362 100644 --- a/airbyte-integrations/connectors/source-us-census/setup.py +++ b/airbyte-integrations/connectors/source-us-census/setup.py @@ -17,6 +17,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-us-census=source_us_census.run:run", + ], + }, name="source_us_census", description="Source implementation for Us Census.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-us-census/source_us_census/run.py b/airbyte-integrations/connectors/source-us-census/source_us_census/run.py new file mode 100644 index 000000000000..20b8b7eedd0c --- /dev/null +++ b/airbyte-integrations/connectors/source-us-census/source_us_census/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_us_census import SourceUsCensus + + +def run(): + source = SourceUsCensus() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-vantage/main.py b/airbyte-integrations/connectors/source-vantage/main.py index fdec95252d78..eab761b3302d 100644 --- a/airbyte-integrations/connectors/source-vantage/main.py +++ b/airbyte-integrations/connectors/source-vantage/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_vantage import SourceVantage +from source_vantage.run import run if __name__ == "__main__": - source = SourceVantage() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-vantage/setup.py b/airbyte-integrations/connectors/source-vantage/setup.py index 11fe68724009..6f3c091a99c3 100644 --- a/airbyte-integrations/connectors/source-vantage/setup.py +++ b/airbyte-integrations/connectors/source-vantage/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-vantage=source_vantage.run:run", + ], + }, name="source_vantage", description="Source implementation for Vantage.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-vantage/source_vantage/run.py b/airbyte-integrations/connectors/source-vantage/source_vantage/run.py new file mode 100644 index 000000000000..a9c2ee3eb36a --- /dev/null +++ b/airbyte-integrations/connectors/source-vantage/source_vantage/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_vantage import SourceVantage + + +def run(): + source = SourceVantage() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-visma-economic/main.py b/airbyte-integrations/connectors/source-visma-economic/main.py index f5a5d1e8f883..ae896df1e48f 100644 --- a/airbyte-integrations/connectors/source-visma-economic/main.py +++ b/airbyte-integrations/connectors/source-visma-economic/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_visma_economic import SourceVismaEconomic +from source_visma_economic.run import run if __name__ == "__main__": - source = SourceVismaEconomic() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-visma-economic/setup.py b/airbyte-integrations/connectors/source-visma-economic/setup.py index 9574bffc86ff..aca046a78f08 100644 --- a/airbyte-integrations/connectors/source-visma-economic/setup.py +++ b/airbyte-integrations/connectors/source-visma-economic/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-visma-economic=source_visma_economic.run:run", + ], + }, name="source_visma_economic", description="Source implementation for Visma Economic.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/run.py b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/run.py new file mode 100644 index 000000000000..4c8146aa54ec --- /dev/null +++ b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_visma_economic import SourceVismaEconomic + + +def run(): + source = SourceVismaEconomic() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-vitally/main.py b/airbyte-integrations/connectors/source-vitally/main.py index d03f28ec1f1b..1410ead03c39 100644 --- a/airbyte-integrations/connectors/source-vitally/main.py +++ b/airbyte-integrations/connectors/source-vitally/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_vitally import SourceVitally +from source_vitally.run import run if __name__ == "__main__": - source = SourceVitally() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-vitally/setup.py b/airbyte-integrations/connectors/source-vitally/setup.py index f7091d2171d5..d0adc3fc2c1f 100644 --- a/airbyte-integrations/connectors/source-vitally/setup.py +++ b/airbyte-integrations/connectors/source-vitally/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-vitally=source_vitally.run:run", + ], + }, name="source_vitally", description="Source implementation for Vitally.", author="Elliot Trabac", author_email="elliot.trabac1@gmail.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-vitally/source_vitally/run.py b/airbyte-integrations/connectors/source-vitally/source_vitally/run.py new file mode 100644 index 000000000000..1c6c2f841ed9 --- /dev/null +++ b/airbyte-integrations/connectors/source-vitally/source_vitally/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_vitally import SourceVitally + + +def run(): + source = SourceVitally() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-waiteraid/main.py b/airbyte-integrations/connectors/source-waiteraid/main.py index 0ba9d249cc5a..50171148de01 100644 --- a/airbyte-integrations/connectors/source-waiteraid/main.py +++ b/airbyte-integrations/connectors/source-waiteraid/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_waiteraid import SourceWaiteraid +from source_waiteraid.run import run if __name__ == "__main__": - source = SourceWaiteraid() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-waiteraid/setup.py b/airbyte-integrations/connectors/source-waiteraid/setup.py index fe5d754e3df5..e2dc21a8895f 100644 --- a/airbyte-integrations/connectors/source-waiteraid/setup.py +++ b/airbyte-integrations/connectors/source-waiteraid/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-waiteraid=source_waiteraid.run:run", + ], + }, name="source_waiteraid", description="Source implementation for Waiteraid.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/run.py b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/run.py new file mode 100644 index 000000000000..5219abc38724 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_waiteraid import SourceWaiteraid + + +def run(): + source = SourceWaiteraid() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-weatherstack/main.py b/airbyte-integrations/connectors/source-weatherstack/main.py index 1924e100bb8d..83c59afceaf5 100644 --- a/airbyte-integrations/connectors/source-weatherstack/main.py +++ b/airbyte-integrations/connectors/source-weatherstack/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_weatherstack import SourceWeatherstack +from source_weatherstack.run import run if __name__ == "__main__": - source = SourceWeatherstack() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-weatherstack/setup.py b/airbyte-integrations/connectors/source-weatherstack/setup.py index 98887751b488..207fe4d3c13d 100644 --- a/airbyte-integrations/connectors/source-weatherstack/setup.py +++ b/airbyte-integrations/connectors/source-weatherstack/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-weatherstack=source_weatherstack.run:run", + ], + }, name="source_weatherstack", description="Source implementation for Weatherstack.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-weatherstack/source_weatherstack/run.py b/airbyte-integrations/connectors/source-weatherstack/source_weatherstack/run.py new file mode 100644 index 000000000000..07f3930dd2b2 --- /dev/null +++ b/airbyte-integrations/connectors/source-weatherstack/source_weatherstack/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_weatherstack import SourceWeatherstack + + +def run(): + source = SourceWeatherstack() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-webflow/main.py b/airbyte-integrations/connectors/source-webflow/main.py index f35348e338c3..4d481e07151b 100644 --- a/airbyte-integrations/connectors/source-webflow/main.py +++ b/airbyte-integrations/connectors/source-webflow/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_webflow import SourceWebflow +from source_webflow.run import run if __name__ == "__main__": - source = SourceWebflow() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-webflow/setup.py b/airbyte-integrations/connectors/source-webflow/setup.py index 6912dcae7d7c..1d604e8b4ea1 100644 --- a/airbyte-integrations/connectors/source-webflow/setup.py +++ b/airbyte-integrations/connectors/source-webflow/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-webflow=source_webflow.run:run", + ], + }, name="source_webflow", description="Source implementation for Webflow.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-webflow/source_webflow/run.py b/airbyte-integrations/connectors/source-webflow/source_webflow/run.py new file mode 100644 index 000000000000..a71297a1d6a2 --- /dev/null +++ b/airbyte-integrations/connectors/source-webflow/source_webflow/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_webflow import SourceWebflow + + +def run(): + source = SourceWebflow() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-whisky-hunter/main.py b/airbyte-integrations/connectors/source-whisky-hunter/main.py index b9e7c8a1c682..76cf72379579 100644 --- a/airbyte-integrations/connectors/source-whisky-hunter/main.py +++ b/airbyte-integrations/connectors/source-whisky-hunter/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_whisky_hunter import SourceWhiskyHunter +from source_whisky_hunter.run import run if __name__ == "__main__": - source = SourceWhiskyHunter() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-whisky-hunter/setup.py b/airbyte-integrations/connectors/source-whisky-hunter/setup.py index 6ea55fe522d9..6b48b33f68c8 100644 --- a/airbyte-integrations/connectors/source-whisky-hunter/setup.py +++ b/airbyte-integrations/connectors/source-whisky-hunter/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-whisky-hunter=source_whisky_hunter.run:run", + ], + }, name="source_whisky_hunter", description="Source implementation for Whisky Hunter.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/run.py b/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/run.py new file mode 100644 index 000000000000..2af9bbe54e12 --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_whisky_hunter import SourceWhiskyHunter + + +def run(): + source = SourceWhiskyHunter() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-wikipedia-pageviews/main.py b/airbyte-integrations/connectors/source-wikipedia-pageviews/main.py index 3f284fc2e06f..1ba1f8403240 100755 --- a/airbyte-integrations/connectors/source-wikipedia-pageviews/main.py +++ b/airbyte-integrations/connectors/source-wikipedia-pageviews/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_wikipedia_pageviews import SourceWikipediaPageviews +from source_wikipedia_pageviews.run import run if __name__ == "__main__": - source = SourceWikipediaPageviews() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-wikipedia-pageviews/setup.py b/airbyte-integrations/connectors/source-wikipedia-pageviews/setup.py index 9716b7daa4ee..f946706e21f9 100755 --- a/airbyte-integrations/connectors/source-wikipedia-pageviews/setup.py +++ b/airbyte-integrations/connectors/source-wikipedia-pageviews/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-wikipedia-pageviews=source_wikipedia_pageviews.run:run", + ], + }, name="source_wikipedia_pageviews", description="Source implementation for Wikipedia Pageviews.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-wikipedia-pageviews/source_wikipedia_pageviews/run.py b/airbyte-integrations/connectors/source-wikipedia-pageviews/source_wikipedia_pageviews/run.py new file mode 100755 index 000000000000..d77093ecb81b --- /dev/null +++ b/airbyte-integrations/connectors/source-wikipedia-pageviews/source_wikipedia_pageviews/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_wikipedia_pageviews import SourceWikipediaPageviews + + +def run(): + source = SourceWikipediaPageviews() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-woocommerce/main.py b/airbyte-integrations/connectors/source-woocommerce/main.py index 606cdff539ba..1fae4df4545e 100644 --- a/airbyte-integrations/connectors/source-woocommerce/main.py +++ b/airbyte-integrations/connectors/source-woocommerce/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_woocommerce import SourceWoocommerce +from source_woocommerce.run import run if __name__ == "__main__": - source = SourceWoocommerce() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-woocommerce/setup.py b/airbyte-integrations/connectors/source-woocommerce/setup.py index 40945a630f15..c69b2f251935 100644 --- a/airbyte-integrations/connectors/source-woocommerce/setup.py +++ b/airbyte-integrations/connectors/source-woocommerce/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-woocommerce=source_woocommerce.run:run", + ], + }, name="source_woocommerce", description="Source implementation for Woocommerce.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-woocommerce/source_woocommerce/run.py b/airbyte-integrations/connectors/source-woocommerce/source_woocommerce/run.py new file mode 100644 index 000000000000..b53f1bed41bb --- /dev/null +++ b/airbyte-integrations/connectors/source-woocommerce/source_woocommerce/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_woocommerce import SourceWoocommerce + + +def run(): + source = SourceWoocommerce() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-workable/main.py b/airbyte-integrations/connectors/source-workable/main.py index 9ca2a8507540..7c63dc17331b 100644 --- a/airbyte-integrations/connectors/source-workable/main.py +++ b/airbyte-integrations/connectors/source-workable/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_workable import SourceWorkable +from source_workable.run import run if __name__ == "__main__": - source = SourceWorkable() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-workable/setup.py b/airbyte-integrations/connectors/source-workable/setup.py index 67f123878709..e35ab25501c6 100644 --- a/airbyte-integrations/connectors/source-workable/setup.py +++ b/airbyte-integrations/connectors/source-workable/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-workable=source_workable.run:run", + ], + }, name="source_workable", description="Source implementation for Workable.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-workable/source_workable/run.py b/airbyte-integrations/connectors/source-workable/source_workable/run.py new file mode 100644 index 000000000000..5b838949c4ef --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/source_workable/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_workable import SourceWorkable + + +def run(): + source = SourceWorkable() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-workramp/main.py b/airbyte-integrations/connectors/source-workramp/main.py index c18f064ccead..7b2a20cedcac 100644 --- a/airbyte-integrations/connectors/source-workramp/main.py +++ b/airbyte-integrations/connectors/source-workramp/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_workramp import SourceWorkramp +from source_workramp.run import run if __name__ == "__main__": - source = SourceWorkramp() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-workramp/setup.py b/airbyte-integrations/connectors/source-workramp/setup.py index a2b4fc952a19..eb12a1398b9c 100644 --- a/airbyte-integrations/connectors/source-workramp/setup.py +++ b/airbyte-integrations/connectors/source-workramp/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-workramp=source_workramp.run:run", + ], + }, name="source_workramp", description="Source implementation for Workramp.", author="Elliot Trabac", author_email="elliot.trabac1@gmail.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-workramp/source_workramp/run.py b/airbyte-integrations/connectors/source-workramp/source_workramp/run.py new file mode 100644 index 000000000000..6b56635b93cb --- /dev/null +++ b/airbyte-integrations/connectors/source-workramp/source_workramp/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_workramp import SourceWorkramp + + +def run(): + source = SourceWorkramp() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-wrike/main.py b/airbyte-integrations/connectors/source-wrike/main.py index 5f853be26873..4c1d0cd0f5fa 100644 --- a/airbyte-integrations/connectors/source-wrike/main.py +++ b/airbyte-integrations/connectors/source-wrike/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_wrike import SourceWrike +from source_wrike.run import run if __name__ == "__main__": - source = SourceWrike() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-wrike/setup.py b/airbyte-integrations/connectors/source-wrike/setup.py index a3c314147ca7..63714da0ca09 100644 --- a/airbyte-integrations/connectors/source-wrike/setup.py +++ b/airbyte-integrations/connectors/source-wrike/setup.py @@ -12,13 +12,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-wrike=source_wrike.run:run", + ], + }, name="source_wrike", description="Source implementation for Wrike.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-wrike/source_wrike/run.py b/airbyte-integrations/connectors/source-wrike/source_wrike/run.py new file mode 100644 index 000000000000..5795da526154 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/source_wrike/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_wrike import SourceWrike + + +def run(): + source = SourceWrike() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-xero/setup.py b/airbyte-integrations/connectors/source-xero/setup.py index 89541436cfd3..31cd43ef5838 100644 --- a/airbyte-integrations/connectors/source-xero/setup.py +++ b/airbyte-integrations/connectors/source-xero/setup.py @@ -27,7 +27,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-xkcd/main.py b/airbyte-integrations/connectors/source-xkcd/main.py index c0de98487459..ea20bdf50104 100644 --- a/airbyte-integrations/connectors/source-xkcd/main.py +++ b/airbyte-integrations/connectors/source-xkcd/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_xkcd import SourceXkcd +from source_xkcd.run import run if __name__ == "__main__": - source = SourceXkcd() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-xkcd/setup.py b/airbyte-integrations/connectors/source-xkcd/setup.py index 7fe43ebd17d6..f6c8db1ffa16 100644 --- a/airbyte-integrations/connectors/source-xkcd/setup.py +++ b/airbyte-integrations/connectors/source-xkcd/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-xkcd=source_xkcd.run:run", + ], + }, name="source_xkcd", description="Source implementation for Xkcd.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-xkcd/source_xkcd/run.py b/airbyte-integrations/connectors/source-xkcd/source_xkcd/run.py new file mode 100644 index 000000000000..3e056bd11961 --- /dev/null +++ b/airbyte-integrations/connectors/source-xkcd/source_xkcd/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_xkcd import SourceXkcd + + +def run(): + source = SourceXkcd() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/main.py b/airbyte-integrations/connectors/source-yahoo-finance-price/main.py index 7f376eec167c..3b00718b8391 100644 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/main.py +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_yahoo_finance_price import SourceYahooFinancePrice +from source_yahoo_finance_price.run import run if __name__ == "__main__": - source = SourceYahooFinancePrice() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/setup.py b/airbyte-integrations/connectors/source-yahoo-finance-price/setup.py index 862dc4e0083d..e010bdc6daee 100644 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/setup.py +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-yahoo-finance-price=source_yahoo_finance_price.run:run", + ], + }, name="source_yahoo_finance_price", description="Source implementation for Yahoo Finance Price.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/source_yahoo_finance_price/run.py b/airbyte-integrations/connectors/source-yahoo-finance-price/source_yahoo_finance_price/run.py new file mode 100644 index 000000000000..d24a645a4183 --- /dev/null +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/source_yahoo_finance_price/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_yahoo_finance_price import SourceYahooFinancePrice + + +def run(): + source = SourceYahooFinancePrice() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-yandex-metrica/main.py b/airbyte-integrations/connectors/source-yandex-metrica/main.py index 6c6a7edcc353..a84b23e0a261 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/main.py +++ b/airbyte-integrations/connectors/source-yandex-metrica/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_yandex_metrica import SourceYandexMetrica +from source_yandex_metrica.run import run if __name__ == "__main__": - source = SourceYandexMetrica() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-yandex-metrica/setup.py b/airbyte-integrations/connectors/source-yandex-metrica/setup.py index ce5d5b361010..adebbc0e9bb2 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/setup.py +++ b/airbyte-integrations/connectors/source-yandex-metrica/setup.py @@ -10,13 +10,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "freezegun", "pytest~=6.1", "pytest-mock", "requests_mock"] setup( + entry_points={ + "console_scripts": [ + "source-yandex-metrica=source_yandex_metrica.run:run", + ], + }, name="source_yandex_metrica", description="Source implementation for Yandex Metrica.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/run.py b/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/run.py new file mode 100644 index 000000000000..69e34c3d1740 --- /dev/null +++ b/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_yandex_metrica import SourceYandexMetrica + + +def run(): + source = SourceYandexMetrica() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-yotpo/main.py b/airbyte-integrations/connectors/source-yotpo/main.py index d456ca2abab3..968a262f7f26 100644 --- a/airbyte-integrations/connectors/source-yotpo/main.py +++ b/airbyte-integrations/connectors/source-yotpo/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_yotpo import SourceYotpo +from source_yotpo.run import run if __name__ == "__main__": - source = SourceYotpo() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-yotpo/setup.py b/airbyte-integrations/connectors/source-yotpo/setup.py index ed41819fb6d9..4d1d2163dbbd 100644 --- a/airbyte-integrations/connectors/source-yotpo/setup.py +++ b/airbyte-integrations/connectors/source-yotpo/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-yotpo=source_yotpo.run:run", + ], + }, name="source_yotpo", description="Source implementation for Yotpo.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-yotpo/source_yotpo/run.py b/airbyte-integrations/connectors/source-yotpo/source_yotpo/run.py new file mode 100644 index 000000000000..d302e5b2f4dd --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/source_yotpo/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_yotpo import SourceYotpo + + +def run(): + source = SourceYotpo() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-younium/main.py b/airbyte-integrations/connectors/source-younium/main.py index 311b077eb084..8fcc0a655630 100644 --- a/airbyte-integrations/connectors/source-younium/main.py +++ b/airbyte-integrations/connectors/source-younium/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_younium import SourceYounium +from source_younium.run import run if __name__ == "__main__": - source = SourceYounium() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-younium/setup.py b/airbyte-integrations/connectors/source-younium/setup.py index 2a8872be5287..25ca4c963925 100644 --- a/airbyte-integrations/connectors/source-younium/setup.py +++ b/airbyte-integrations/connectors/source-younium/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-younium=source_younium.run:run", + ], + }, name="source_younium", description="Source implementation for Younium.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-younium/source_younium/run.py b/airbyte-integrations/connectors/source-younium/source_younium/run.py new file mode 100644 index 000000000000..5250f8d9bd94 --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/source_younium/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_younium import SourceYounium + + +def run(): + source = SourceYounium() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-youtube-analytics/main.py b/airbyte-integrations/connectors/source-youtube-analytics/main.py index 046af990bcb0..f2542cccc965 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/main.py +++ b/airbyte-integrations/connectors/source-youtube-analytics/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_youtube_analytics import SourceYoutubeAnalytics +from source_youtube_analytics.run import run if __name__ == "__main__": - source = SourceYoutubeAnalytics() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-youtube-analytics/setup.py b/airbyte-integrations/connectors/source-youtube-analytics/setup.py index 795c2044a1ca..9b17c4365ad8 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/setup.py +++ b/airbyte-integrations/connectors/source-youtube-analytics/setup.py @@ -16,6 +16,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-youtube-analytics=source_youtube_analytics.run:run", + ], + }, name="source_youtube_analytics", description="Source implementation for Youtube Analytics.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-youtube-analytics/source_youtube_analytics/run.py b/airbyte-integrations/connectors/source-youtube-analytics/source_youtube_analytics/run.py new file mode 100644 index 000000000000..27f8967eff43 --- /dev/null +++ b/airbyte-integrations/connectors/source-youtube-analytics/source_youtube_analytics/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_youtube_analytics import SourceYoutubeAnalytics + + +def run(): + source = SourceYoutubeAnalytics() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zapier-supported-storage/main.py b/airbyte-integrations/connectors/source-zapier-supported-storage/main.py index b65c91f88848..8de0f27f55af 100644 --- a/airbyte-integrations/connectors/source-zapier-supported-storage/main.py +++ b/airbyte-integrations/connectors/source-zapier-supported-storage/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_zapier_supported_storage import SourceZapierSupportedStorage +from source_zapier_supported_storage.run import run if __name__ == "__main__": - source = SourceZapierSupportedStorage() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-zapier-supported-storage/setup.py b/airbyte-integrations/connectors/source-zapier-supported-storage/setup.py index 1ffccc828857..9feb0c8cde94 100644 --- a/airbyte-integrations/connectors/source-zapier-supported-storage/setup.py +++ b/airbyte-integrations/connectors/source-zapier-supported-storage/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-zapier-supported-storage=source_zapier_supported_storage.run:run", + ], + }, name="source_zapier_supported_storage", description="Source implementation for Zapier Supported Storage.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-zapier-supported-storage/source_zapier_supported_storage/run.py b/airbyte-integrations/connectors/source-zapier-supported-storage/source_zapier_supported_storage/run.py new file mode 100644 index 000000000000..a3777475b5b8 --- /dev/null +++ b/airbyte-integrations/connectors/source-zapier-supported-storage/source_zapier_supported_storage/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zapier_supported_storage import SourceZapierSupportedStorage + + +def run(): + source = SourceZapierSupportedStorage() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/main.py b/airbyte-integrations/connectors/source-zendesk-sell/main.py index 6f01a3b8f7a8..e57dccf7d35e 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/main.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_zendesk_sell import SourceZendeskSell +from source_zendesk_sell.run import run if __name__ == "__main__": - source = SourceZendeskSell() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-zendesk-sell/setup.py b/airbyte-integrations/connectors/source-zendesk-sell/setup.py index 054742fb41d4..a789b08aeb99 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/setup.py @@ -14,13 +14,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-zendesk-sell=source_zendesk_sell.run:run", + ], + }, name="source_zendesk_sell", description="Source implementation for Zendesk Sell.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/run.py b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/run.py new file mode 100644 index 000000000000..0e238308da46 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zendesk_sell import SourceZendeskSell + + +def run(): + source = SourceZendeskSell() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/main.py b/airbyte-integrations/connectors/source-zendesk-sunshine/main.py index 0a86066993fc..4b7507ee396c 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/main.py +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_zendesk_sunshine import SourceZendeskSunshine +from source_zendesk_sunshine.run import run if __name__ == "__main__": - source = SourceZendeskSunshine() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/setup.py b/airbyte-integrations/connectors/source-zendesk-sunshine/setup.py index 1c47ce8ab0a9..d59a2a4793b4 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-zendesk-sunshine=source_zendesk_sunshine.run:run", + ], + }, name="source_zendesk_sunshine", description="Source implementation for Zendesk Sunshine.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/run.py b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/run.py new file mode 100644 index 000000000000..be323283bc95 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zendesk_sunshine import SourceZendeskSunshine + + +def run(): + source = SourceZendeskSunshine() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zenefits/main.py b/airbyte-integrations/connectors/source-zenefits/main.py index 480734a07faa..7124907d176d 100644 --- a/airbyte-integrations/connectors/source-zenefits/main.py +++ b/airbyte-integrations/connectors/source-zenefits/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_zenefits import SourceZenefits +from source_zenefits.run import run if __name__ == "__main__": - source = SourceZenefits() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-zenefits/setup.py b/airbyte-integrations/connectors/source-zenefits/setup.py index 5cc7df205794..7d33a0d79c91 100644 --- a/airbyte-integrations/connectors/source-zenefits/setup.py +++ b/airbyte-integrations/connectors/source-zenefits/setup.py @@ -12,13 +12,30 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-zenefits=source_zenefits.run:run", + ], + }, name="source_zenefits", description="Source implementation for Zenefits.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-zenefits/source_zenefits/run.py b/airbyte-integrations/connectors/source-zenefits/source_zenefits/run.py new file mode 100644 index 000000000000..3fa66926fadf --- /dev/null +++ b/airbyte-integrations/connectors/source-zenefits/source_zenefits/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zenefits import SourceZenefits + + +def run(): + source = SourceZenefits() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zenloop/main.py b/airbyte-integrations/connectors/source-zenloop/main.py index ec861621c9c7..dd3a6687740e 100644 --- a/airbyte-integrations/connectors/source-zenloop/main.py +++ b/airbyte-integrations/connectors/source-zenloop/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_zenloop import SourceZenloop +from source_zenloop.run import run if __name__ == "__main__": - source = SourceZenloop() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-zenloop/setup.py b/airbyte-integrations/connectors/source-zenloop/setup.py index be53c38ee558..f6920042865b 100644 --- a/airbyte-integrations/connectors/source-zenloop/setup.py +++ b/airbyte-integrations/connectors/source-zenloop/setup.py @@ -17,6 +17,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-zenloop=source_zenloop.run:run", + ], + }, name="source_zenloop", description="Source implementation for Zenloop.", author="Alexander Batoulis", diff --git a/airbyte-integrations/connectors/source-zenloop/source_zenloop/run.py b/airbyte-integrations/connectors/source-zenloop/source_zenloop/run.py new file mode 100644 index 000000000000..344453ff360c --- /dev/null +++ b/airbyte-integrations/connectors/source-zenloop/source_zenloop/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zenloop import SourceZenloop + + +def run(): + source = SourceZenloop() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zoho-crm/main.py b/airbyte-integrations/connectors/source-zoho-crm/main.py index 911fb44f52f2..2cf82bb23bf9 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/main.py +++ b/airbyte-integrations/connectors/source-zoho-crm/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_zoho_crm import SourceZohoCrm +from source_zoho_crm.run import run if __name__ == "__main__": - source = SourceZohoCrm() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-zoho-crm/setup.py b/airbyte-integrations/connectors/source-zoho-crm/setup.py index 4627c0115e63..a44e9b0529a7 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/setup.py +++ b/airbyte-integrations/connectors/source-zoho-crm/setup.py @@ -16,6 +16,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-zoho-crm=source_zoho_crm.run:run", + ], + }, name="source_zoho_crm", description="Source implementation for Zoho Crm.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/run.py b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/run.py new file mode 100644 index 000000000000..d915dc05f2b9 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zoho_crm import SourceZohoCrm + + +def run(): + source = SourceZohoCrm() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zoom/main.py b/airbyte-integrations/connectors/source-zoom/main.py index da33508f6a8e..137aea5931e6 100644 --- a/airbyte-integrations/connectors/source-zoom/main.py +++ b/airbyte-integrations/connectors/source-zoom/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_zoom import SourceZoom +from source_zoom.run import run if __name__ == "__main__": - source = SourceZoom() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-zoom/setup.py b/airbyte-integrations/connectors/source-zoom/setup.py index edc76fde557e..6d4f526d5e35 100644 --- a/airbyte-integrations/connectors/source-zoom/setup.py +++ b/airbyte-integrations/connectors/source-zoom/setup.py @@ -16,13 +16,30 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-zoom=source_zoom.run:run", + ], + }, name="source_zoom", description="Source implementation for Zoom.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/run.py b/airbyte-integrations/connectors/source-zoom/source_zoom/run.py new file mode 100644 index 000000000000..e663e8441844 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zoom import SourceZoom + + +def run(): + source = SourceZoom() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zuora/main.py b/airbyte-integrations/connectors/source-zuora/main.py index e65d47a77eda..404a72854a9e 100644 --- a/airbyte-integrations/connectors/source-zuora/main.py +++ b/airbyte-integrations/connectors/source-zuora/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_zuora import SourceZuora +from source_zuora.run import run if __name__ == "__main__": - source = SourceZuora() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-zuora/setup.py b/airbyte-integrations/connectors/source-zuora/setup.py index 6fb6e922ce98..43b397e5c183 100644 --- a/airbyte-integrations/connectors/source-zuora/setup.py +++ b/airbyte-integrations/connectors/source-zuora/setup.py @@ -16,6 +16,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-zuora=source_zuora.run:run", + ], + }, name="source_zuora", description="Airbyte source-connector for Zuora.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-zuora/source_zuora/run.py b/airbyte-integrations/connectors/source-zuora/source_zuora/run.py new file mode 100644 index 000000000000..58495d4d67e3 --- /dev/null +++ b/airbyte-integrations/connectors/source-zuora/source_zuora/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zuora import SourceZuora + + +def run(): + source = SourceZuora() + launch(source, sys.argv[1:]) From bd0a8774dc506675451e84a9bffab8e0914b4daf Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 30 Jan 2024 10:35:30 +0100 Subject: [PATCH 14/96] airbyte-lib: Fix validation (#34599) --- airbyte-lib/airbyte_lib/validate.py | 2 +- .../tests/integration_tests/fixtures/source-broken/setup.py | 2 +- .../tests/integration_tests/fixtures/source-test/setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-lib/airbyte_lib/validate.py b/airbyte-lib/airbyte_lib/validate.py index 551960ed5250..306b2b58a012 100644 --- a/airbyte-lib/airbyte_lib/validate.py +++ b/airbyte-lib/airbyte_lib/validate.py @@ -129,7 +129,7 @@ def validate(connector_dir: str, sample_config: str, *, validate_install_only: b pip_path = str(venv_path / "bin" / "pip") - _run_subprocess_and_raise_on_failure([pip_path, "install", "-e", connector_dir]) + _run_subprocess_and_raise_on_failure([pip_path, "install", connector_dir]) # write basic registry to temp json file registry = { diff --git a/airbyte-lib/tests/integration_tests/fixtures/source-broken/setup.py b/airbyte-lib/tests/integration_tests/fixtures/source-broken/setup.py index 1172b397f493..0a565f98e08b 100644 --- a/airbyte-lib/tests/integration_tests/fixtures/source-broken/setup.py +++ b/airbyte-lib/tests/integration_tests/fixtures/source-broken/setup.py @@ -11,7 +11,7 @@ description="Test Soutce", author="Airbyte", author_email="contact@airbyte.io", - packages=find_packages(), + packages=["source_broken"], entry_points={ "console_scripts": [ "source-broken=source_broken.run:run", diff --git a/airbyte-lib/tests/integration_tests/fixtures/source-test/setup.py b/airbyte-lib/tests/integration_tests/fixtures/source-test/setup.py index b59aca2ec5c3..348e0c28a254 100644 --- a/airbyte-lib/tests/integration_tests/fixtures/source-test/setup.py +++ b/airbyte-lib/tests/integration_tests/fixtures/source-test/setup.py @@ -11,7 +11,7 @@ description="Test Soutce", author="Airbyte", author_email="contact@airbyte.io", - packages=find_packages(), + packages=["source_test"], entry_points={ "console_scripts": [ "source-test=source_test.run:run", From b55173292182a13aafe4ec88c7cba39f7ee12a73 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 30 Jan 2024 10:35:42 +0100 Subject: [PATCH 15/96] =?UTF-8?q?=F0=9F=93=9DDestination=20Astra=20DB=20Co?= =?UTF-8?q?nnector=20Name=20Update,=20Icon=20Update,=20Spec=20formatting?= =?UTF-8?q?=20and=20tooltips=20(#34600)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Obioma Anomnachi --- .../destination_astra/config.py | 20 +++++-- .../connectors/destination-astra/icon.svg | 57 ++++--------------- .../integration_tests/spec.json | 16 +++--- .../destination-astra/metadata.yaml | 4 +- docs/integrations/destinations/astra.md | 3 +- 5 files changed, 38 insertions(+), 62 deletions(-) diff --git a/airbyte-integrations/connectors/destination-astra/destination_astra/config.py b/airbyte-integrations/connectors/destination-astra/destination_astra/config.py index 7606aab4f1c5..01d805ecd782 100644 --- a/airbyte-integrations/connectors/destination-astra/destination_astra/config.py +++ b/airbyte-integrations/connectors/destination-astra/destination_astra/config.py @@ -9,19 +9,27 @@ class AstraIndexingModel(BaseModel): astra_db_app_token: str = Field( ..., - title="AstraDB Application Token", + title="Astra DB Application Token", airbyte_secret=True, - description="AstraDB Application Token", + description="The application token authorizes a user to connect to a specific Astra DB database. It is created when the user clicks the Generate Token button on the Overview tab of the Database page in the Astra UI.", ) astra_db_endpoint: str = Field( ..., - title="AstraDB Endpoint", - description="AstraDB Endpoint", + title="Astra DB Endpoint", + description="The endpoint specifies which Astra DB database queries are sent to. It can be copied from the Database Details section of the Overview tab of the Database page in the Astra UI.", pattern="^https:\\/\\/([a-z]|[0-9]){8}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){12}-[^\\.]*?\\.apps\\.astra\\.datastax\\.com", examples=["https://8292d414-dd1b-4c33-8431-e838bedc04f7-us-east1.apps.astra.datastax.com"], ) - astra_db_keyspace: str = Field(..., title="AstraDB Keyspace", description="Astra DB Keyspace") - collection: str = Field(..., title="AstraDB collection", description="AstraDB collection") + astra_db_keyspace: str = Field( + ..., + title="Astra DB Keyspace", + description="Keyspaces (or Namespaces) serve as containers for organizing data within a database. You can create a new keyspace uisng the Data Explorer tab in the Astra UI. The keyspace default_keyspace is created for you when you create a Vector Database in Astra DB.", + ) + collection: str = Field( + ..., + title="Astra DB collection", + description="Collections hold data. They are analagous to tables in traditional Cassandra terminology. This tool will create the collection with the provided name automatically if it does not already exist. Alternatively, you can create one thorugh the Data Explorer tab in the Astra UI.", + ) class Config: title = "Indexing" diff --git a/airbyte-integrations/connectors/destination-astra/icon.svg b/airbyte-integrations/connectors/destination-astra/icon.svg index 2d1f6c918ed9..ecc353976f51 100644 --- a/airbyte-integrations/connectors/destination-astra/icon.svg +++ b/airbyte-integrations/connectors/destination-astra/icon.svg @@ -1,46 +1,13 @@ - - - - - - - - - + + + + + + + + + + + + diff --git a/airbyte-integrations/connectors/destination-astra/integration_tests/spec.json b/airbyte-integrations/connectors/destination-astra/integration_tests/spec.json index a94caed893cf..35951290a06c 100644 --- a/airbyte-integrations/connectors/destination-astra/integration_tests/spec.json +++ b/airbyte-integrations/connectors/destination-astra/integration_tests/spec.json @@ -317,14 +317,14 @@ "type": "object", "properties": { "astra_db_app_token": { - "title": "AstraDB Application Token", - "description": "AstraDB Application Token", + "title": "Astra DB Application Token", + "description": "The application token authorizes a user to connect to a specific Astra DB database. It is created when the user clicks the Generate Token button on the Overview tab of the Database page in the Astra UI.", "airbyte_secret": true, "type": "string" }, "astra_db_endpoint": { - "title": "AstraDB Endpoint", - "description": "AstraDB Endpoint", + "title": "Astra DB Endpoint", + "description": "The endpoint specifies which Astra DB database queries are sent to. It can be copied from the Database Details section of the Overview tab of the Database page in the Astra UI.", "pattern": "^https:\\/\\/([a-z]|[0-9]){8}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){12}-[^\\.]*?\\.apps\\.astra\\.datastax\\.com", "examples": [ "https://8292d414-dd1b-4c33-8431-e838bedc04f7-us-east1.apps.astra.datastax.com" @@ -332,13 +332,13 @@ "type": "string" }, "astra_db_keyspace": { - "title": "AstraDB Keyspace", - "description": "Astra DB Keyspace", + "title": "Astra DB Keyspace", + "description": "Keyspaces (or Namespaces) serve as containers for organizing data within a database. You can create a new keyspace uisng the Data Explorer tab in the Astra UI. The keyspace default_keyspace is created for you when you create a Vector Database in Astra DB.", "type": "string" }, "collection": { - "title": "AstraDB collection", - "description": "AstraDB collection", + "title": "Astra DB collection", + "description": "Collections hold data. They are analagous to tables in traditional Cassandra terminology. This tool will create the collection with the provided name automatically if it does not already exist. Alternatively, you can create one thorugh the Data Explorer tab in the Astra UI.", "type": "string" } }, diff --git a/airbyte-integrations/connectors/destination-astra/metadata.yaml b/airbyte-integrations/connectors/destination-astra/metadata.yaml index c9c245c358d3..c675ed875fdd 100644 --- a/airbyte-integrations/connectors/destination-astra/metadata.yaml +++ b/airbyte-integrations/connectors/destination-astra/metadata.yaml @@ -15,12 +15,12 @@ data: connectorSubtype: database connectorType: destination definitionId: 042ce96f-1158-4662-9543-e2ff015be97a - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 dockerRepository: airbyte/destination-astra githubIssueLabel: destination-astra icon: astra.svg license: MIT - name: Astra + name: Astra DB releaseDate: 2024-01-10 releaseStage: alpha supportLevel: community diff --git a/docs/integrations/destinations/astra.md b/docs/integrations/destinations/astra.md index 554368588307..b17eb8c29e8c 100644 --- a/docs/integrations/destinations/astra.md +++ b/docs/integrations/destinations/astra.md @@ -1,4 +1,4 @@ -# Astra Destination +# Astra DB Destination This page contains the setup guide and reference information for the destination-astra connector. @@ -38,4 +38,5 @@ This page contains the setup guide and reference information for the destination ## Changelog | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------------- | +| 0.1.1 | 2024-01-26 | | DS Branding Update | | 0.1.0 | 2024-01-08 | | Initial Release | From cc1f1a28590ca3efaf6bea3135d7aaa6a225d38b Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 30 Jan 2024 12:11:41 +0100 Subject: [PATCH 16/96] airbyte-lib: Refactor follow-up (#34649) --- .../connectors/source-airtable/setup.py | 14 +++++++++++++- .../connectors/source-amazon-ads/setup.py | 14 +++++++++++++- .../source-amazon-seller-partner/setup.py | 14 +++++++++++++- .../connectors/source-amazon-sqs/setup.py | 14 +++++++++++++- .../connectors/source-appsflyer/setup.py | 14 +++++++++++++- .../connectors/source-appstore-singer/setup.py | 14 +++++++++++++- .../connectors/source-asana/setup.py | 14 +++++++++++++- .../connectors/source-aws-cloudtrail/setup.py | 14 +++++++++++++- .../connectors/source-azure-blob-storage/setup.py | 14 +++++++++++++- .../connectors/source-azure-table/setup.py | 14 +++++++++++++- .../connectors/source-bamboo-hr/setup.py | 14 +++++++++++++- .../connectors/source-bing-ads/setup.py | 14 +++++++++++++- .../connectors/source-cart/setup.py | 14 +++++++++++++- .../connectors/source-chartmogul/setup.py | 14 +++++++++++++- .../connectors/source-close-com/setup.py | 14 +++++++++++++- .../connectors/source-delighted/setup.py | 14 +++++++++++++- .../connectors/source-dv-360/setup.py | 14 +++++++++++++- .../connectors/source-facebook-marketing/setup.py | 14 +++++++++++++- .../connectors/source-faker/setup.py | 14 +++++++++++++- .../connectors/source-fauna/setup.py | 14 +++++++++++++- .../connectors/source-file/setup.py | 14 +++++++++++++- .../source-firebase-realtime-database/setup.py | 14 +++++++++++++- .../connectors/source-firebolt/setup.py | 14 +++++++++++++- .../connectors/source-freshdesk/setup.py | 14 +++++++++++++- .../connectors/source-gcs/setup.py | 14 +++++++++++++- .../connectors/source-github/setup.py | 14 +++++++++++++- .../connectors/source-gitlab/setup.py | 14 +++++++++++++- .../connectors/source-google-ads/setup.py | 14 +++++++++++++- .../source-google-analytics-data-api/setup.py | 14 +++++++++++++- .../setup.py | 14 +++++++++++++- .../connectors/source-google-analytics-v4/setup.py | 14 +++++++++++++- .../connectors/source-google-directory/setup.py | 14 +++++++++++++- .../connectors/source-google-drive/setup.py | 14 +++++++++++++- .../credentials/setup.py | 14 +++++++++++++- .../source-google-search-console/setup.py | 14 +++++++++++++- .../source-google-workspace-admin-reports/setup.py | 14 +++++++++++++- .../connectors/source-greenhouse/setup.py | 14 +++++++++++++- .../connectors/source-harvest/setup.py | 14 +++++++++++++- .../connectors/source-hubspot/setup.py | 14 +++++++++++++- .../connectors/source-instagram/setup.py | 14 +++++++++++++- .../connectors/source-iterable/setup.py | 14 +++++++++++++- .../connectors/source-jira/setup.py | 14 +++++++++++++- .../connectors/source-klaviyo/setup.py | 14 +++++++++++++- .../connectors/source-kustomer-singer/setup.py | 14 +++++++++++++- .../connectors/source-kyriba/setup.py | 14 +++++++++++++- .../connectors/source-kyve/setup.py | 14 +++++++++++++- .../connectors/source-lever-hiring/setup.py | 14 +++++++++++++- .../connectors/source-linkedin-ads/setup.py | 14 +++++++++++++- .../connectors/source-linkedin-pages/setup.py | 14 +++++++++++++- .../connectors/source-linnworks/setup.py | 14 +++++++++++++- .../connectors/source-looker/setup.py | 14 +++++++++++++- .../connectors/source-mailchimp/setup.py | 14 +++++++++++++- .../connectors/source-marketo/setup.py | 14 +++++++++++++- .../connectors/source-microsoft-teams/setup.py | 14 +++++++++++++- .../connectors/source-mixpanel/setup.py | 14 +++++++++++++- .../connectors/source-monday/setup.py | 14 +++++++++++++- .../connectors/source-my-hours/setup.py | 14 +++++++++++++- .../connectors/source-notion/setup.py | 14 +++++++++++++- .../connectors/source-okta/setup.py | 14 +++++++++++++- .../connectors/source-orb/setup.py | 14 +++++++++++++- .../connectors/source-outreach/setup.py | 14 +++++++++++++- .../connectors/source-pardot/setup.py | 14 +++++++++++++- .../connectors/source-paystack/setup.py | 14 +++++++++++++- .../connectors/source-pinterest/setup.py | 14 +++++++++++++- .../connectors/source-pivotal-tracker/setup.py | 14 +++++++++++++- .../connectors/source-posthog/setup.py | 14 +++++++++++++- .../source-python-http-tutorial/setup.py | 14 +++++++++++++- .../connectors/source-recharge/setup.py | 14 +++++++++++++- .../connectors/source-recurly/setup.py | 14 +++++++++++++- .../connectors/source-rki-covid/setup.py | 14 +++++++++++++- airbyte-integrations/connectors/source-s3/setup.py | 14 +++++++++++++- .../connectors/source-salesloft/setup.py | 14 +++++++++++++- .../source-scaffold-source-python/setup.py | 14 +++++++++++++- .../connectors/source-search-metrics/setup.py | 14 +++++++++++++- .../connectors/source-sendgrid/setup.py | 14 +++++++++++++- .../connectors/source-sentry/setup.py | 14 +++++++++++++- .../connectors/source-sftp-bulk/setup.py | 14 +++++++++++++- .../connectors/source-shopify/setup.py | 14 +++++++++++++- .../connectors/source-slack/setup.py | 14 +++++++++++++- .../connectors/source-smartsheets/setup.py | 14 +++++++++++++- .../connectors/source-snapchat-marketing/setup.py | 14 +++++++++++++- .../connectors/source-surveymonkey/setup.py | 14 +++++++++++++- .../connectors/source-talkdesk-explore/setup.py | 14 +++++++++++++- .../connectors/source-tempo/setup.py | 14 +++++++++++++- .../connectors/source-tiktok-marketing/setup.py | 14 +++++++++++++- .../connectors/source-tplcentral/setup.py | 14 +++++++++++++- .../connectors/source-twilio/setup.py | 14 +++++++++++++- .../connectors/source-us-census/setup.py | 14 +++++++++++++- .../connectors/source-youtube-analytics/setup.py | 14 +++++++++++++- .../connectors/source-zendesk-chat/setup.py | 14 +++++++++++++- .../connectors/source-zendesk-support/setup.py | 14 +++++++++++++- .../connectors/source-zendesk-talk/setup.py | 14 +++++++++++++- .../connectors/source-zenloop/setup.py | 14 +++++++++++++- .../connectors/source-zoho-crm/setup.py | 14 +++++++++++++- .../connectors/source-zuora/setup.py | 14 +++++++++++++- 95 files changed, 1235 insertions(+), 95 deletions(-) diff --git a/airbyte-integrations/connectors/source-airtable/setup.py b/airbyte-integrations/connectors/source-airtable/setup.py index 9cae02970539..0c9059006741 100644 --- a/airbyte-integrations/connectors/source-airtable/setup.py +++ b/airbyte-integrations/connectors/source-airtable/setup.py @@ -27,7 +27,19 @@ author_email="anhtuan.nguyen@me.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-amazon-ads/setup.py b/airbyte-integrations/connectors/source-amazon-ads/setup.py index 18026e193950..d0d29152fbce 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/setup.py +++ b/airbyte-integrations/connectors/source-amazon-ads/setup.py @@ -28,7 +28,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py b/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py index 2b9f48ca5e81..ffb08022871d 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py @@ -21,7 +21,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-amazon-sqs/setup.py b/airbyte-integrations/connectors/source-amazon-sqs/setup.py index 12414e180650..e39e0d894b21 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/setup.py +++ b/airbyte-integrations/connectors/source-amazon-sqs/setup.py @@ -21,7 +21,19 @@ author_email="airbyte@alasdairb.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-appsflyer/setup.py b/airbyte-integrations/connectors/source-appsflyer/setup.py index cabee5977060..613efc02fdda 100644 --- a/airbyte-integrations/connectors/source-appsflyer/setup.py +++ b/airbyte-integrations/connectors/source-appsflyer/setup.py @@ -25,7 +25,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-appstore-singer/setup.py b/airbyte-integrations/connectors/source-appstore-singer/setup.py index d85ef5d07f43..ecf51e5403a8 100644 --- a/airbyte-integrations/connectors/source-appstore-singer/setup.py +++ b/airbyte-integrations/connectors/source-appstore-singer/setup.py @@ -30,7 +30,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-asana/setup.py b/airbyte-integrations/connectors/source-asana/setup.py index 7d19a10aad8a..08e8edc8363f 100644 --- a/airbyte-integrations/connectors/source-asana/setup.py +++ b/airbyte-integrations/connectors/source-asana/setup.py @@ -23,7 +23,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/setup.py b/airbyte-integrations/connectors/source-aws-cloudtrail/setup.py index 77d6bdf20486..3bf29110ccb4 100644 --- a/airbyte-integrations/connectors/source-aws-cloudtrail/setup.py +++ b/airbyte-integrations/connectors/source-aws-cloudtrail/setup.py @@ -25,7 +25,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/setup.py b/airbyte-integrations/connectors/source-azure-blob-storage/setup.py index 1dc7c4a275c4..b22b1d7c6be6 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/setup.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/setup.py @@ -25,7 +25,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-azure-table/setup.py b/airbyte-integrations/connectors/source-azure-table/setup.py index 31eceef9c80e..2a7451ab9484 100644 --- a/airbyte-integrations/connectors/source-azure-table/setup.py +++ b/airbyte-integrations/connectors/source-azure-table/setup.py @@ -25,7 +25,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-bamboo-hr/setup.py b/airbyte-integrations/connectors/source-bamboo-hr/setup.py index 52914c4c34d4..465c981987f3 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/setup.py +++ b/airbyte-integrations/connectors/source-bamboo-hr/setup.py @@ -25,7 +25,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-bing-ads/setup.py b/airbyte-integrations/connectors/source-bing-ads/setup.py index 68f92f541a5f..131d37a10a3d 100644 --- a/airbyte-integrations/connectors/source-bing-ads/setup.py +++ b/airbyte-integrations/connectors/source-bing-ads/setup.py @@ -26,7 +26,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-cart/setup.py b/airbyte-integrations/connectors/source-cart/setup.py index fe389ea16d47..c0ee59c3d047 100644 --- a/airbyte-integrations/connectors/source-cart/setup.py +++ b/airbyte-integrations/connectors/source-cart/setup.py @@ -27,7 +27,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-chartmogul/setup.py b/airbyte-integrations/connectors/source-chartmogul/setup.py index 57b6c73fbac8..624ab8c53ba2 100644 --- a/airbyte-integrations/connectors/source-chartmogul/setup.py +++ b/airbyte-integrations/connectors/source-chartmogul/setup.py @@ -27,7 +27,19 @@ author_email="titas@omnisend.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-close-com/setup.py b/airbyte-integrations/connectors/source-close-com/setup.py index a67ed8241bb0..a6ad55159cd5 100644 --- a/airbyte-integrations/connectors/source-close-com/setup.py +++ b/airbyte-integrations/connectors/source-close-com/setup.py @@ -25,7 +25,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-delighted/setup.py b/airbyte-integrations/connectors/source-delighted/setup.py index f1f4a7be9d78..fdffdc368480 100644 --- a/airbyte-integrations/connectors/source-delighted/setup.py +++ b/airbyte-integrations/connectors/source-delighted/setup.py @@ -28,7 +28,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-dv-360/setup.py b/airbyte-integrations/connectors/source-dv-360/setup.py index 99188791455c..850d55c1e665 100644 --- a/airbyte-integrations/connectors/source-dv-360/setup.py +++ b/airbyte-integrations/connectors/source-dv-360/setup.py @@ -21,7 +21,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/setup.py b/airbyte-integrations/connectors/source-facebook-marketing/setup.py index 44f12e25a0d1..1c8babe488c3 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/setup.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/setup.py @@ -21,7 +21,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-faker/setup.py b/airbyte-integrations/connectors/source-faker/setup.py index ab39ea239037..be34ee990f61 100644 --- a/airbyte-integrations/connectors/source-faker/setup.py +++ b/airbyte-integrations/connectors/source-faker/setup.py @@ -20,7 +20,19 @@ author_email="evan@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "record_data/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-fauna/setup.py b/airbyte-integrations/connectors/source-fauna/setup.py index db1a676e8035..25c4e60b8647 100644 --- a/airbyte-integrations/connectors/source-fauna/setup.py +++ b/airbyte-integrations/connectors/source-fauna/setup.py @@ -28,7 +28,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-file/setup.py b/airbyte-integrations/connectors/source-file/setup.py index b2ae12af1513..ac9498821d4c 100644 --- a/airbyte-integrations/connectors/source-file/setup.py +++ b/airbyte-integrations/connectors/source-file/setup.py @@ -38,7 +38,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/setup.py b/airbyte-integrations/connectors/source-firebase-realtime-database/setup.py index 84f7e112ff68..780ac7c466df 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/setup.py +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/setup.py @@ -29,7 +29,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-firebolt/setup.py b/airbyte-integrations/connectors/source-firebolt/setup.py index 7c342fd2ae16..3e7be0197a3b 100644 --- a/airbyte-integrations/connectors/source-firebolt/setup.py +++ b/airbyte-integrations/connectors/source-firebolt/setup.py @@ -26,7 +26,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-freshdesk/setup.py b/airbyte-integrations/connectors/source-freshdesk/setup.py index b9cabbadddfc..95c1abd99b17 100644 --- a/airbyte-integrations/connectors/source-freshdesk/setup.py +++ b/airbyte-integrations/connectors/source-freshdesk/setup.py @@ -26,7 +26,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-gcs/setup.py b/airbyte-integrations/connectors/source-gcs/setup.py index f218f69dcb09..0f576ec930cc 100644 --- a/airbyte-integrations/connectors/source-gcs/setup.py +++ b/airbyte-integrations/connectors/source-gcs/setup.py @@ -30,7 +30,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-github/setup.py b/airbyte-integrations/connectors/source-github/setup.py index 822fd9e37edf..f2e77e24ac02 100644 --- a/airbyte-integrations/connectors/source-github/setup.py +++ b/airbyte-integrations/connectors/source-github/setup.py @@ -21,7 +21,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-gitlab/setup.py b/airbyte-integrations/connectors/source-gitlab/setup.py index 2d16bcd7d058..1d006b51ad96 100644 --- a/airbyte-integrations/connectors/source-gitlab/setup.py +++ b/airbyte-integrations/connectors/source-gitlab/setup.py @@ -21,7 +21,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-google-ads/setup.py b/airbyte-integrations/connectors/source-google-ads/setup.py index 7211a092ff54..545e49124b1e 100644 --- a/airbyte-integrations/connectors/source-google-ads/setup.py +++ b/airbyte-integrations/connectors/source-google-ads/setup.py @@ -25,7 +25,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/setup.py b/airbyte-integrations/connectors/source-google-analytics-data-api/setup.py index f2a10ce3101c..ced6eab0083a 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/setup.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/setup.py @@ -26,7 +26,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/setup.py b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/setup.py index c405add054d6..1c9b47c2ec4e 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/setup.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/setup.py @@ -28,7 +28,19 @@ def local_dependency(name: str) -> str: author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/setup.py b/airbyte-integrations/connectors/source-google-analytics-v4/setup.py index 43f18fd04b37..8f5be0996f3c 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/setup.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/setup.py @@ -26,7 +26,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "defaults/*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-google-directory/setup.py b/airbyte-integrations/connectors/source-google-directory/setup.py index e10411bc2668..ac1950ff3ea1 100644 --- a/airbyte-integrations/connectors/source-google-directory/setup.py +++ b/airbyte-integrations/connectors/source-google-directory/setup.py @@ -31,7 +31,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-google-drive/setup.py b/airbyte-integrations/connectors/source-google-drive/setup.py index 1015092ff1c8..0eade80fe5ca 100644 --- a/airbyte-integrations/connectors/source-google-drive/setup.py +++ b/airbyte-integrations/connectors/source-google-drive/setup.py @@ -25,7 +25,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-google-search-console/credentials/setup.py b/airbyte-integrations/connectors/source-google-search-console/credentials/setup.py index 1174b079d6b3..4e39115533b4 100755 --- a/airbyte-integrations/connectors/source-google-search-console/credentials/setup.py +++ b/airbyte-integrations/connectors/source-google-search-console/credentials/setup.py @@ -20,7 +20,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-google-search-console/setup.py b/airbyte-integrations/connectors/source-google-search-console/setup.py index 9888c3a79d5e..791c8b7beba3 100755 --- a/airbyte-integrations/connectors/source-google-search-console/setup.py +++ b/airbyte-integrations/connectors/source-google-search-console/setup.py @@ -30,7 +30,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/setup.py b/airbyte-integrations/connectors/source-google-workspace-admin-reports/setup.py index 787e9f8981b4..9d85298d1b15 100644 --- a/airbyte-integrations/connectors/source-google-workspace-admin-reports/setup.py +++ b/airbyte-integrations/connectors/source-google-workspace-admin-reports/setup.py @@ -32,7 +32,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-greenhouse/setup.py b/airbyte-integrations/connectors/source-greenhouse/setup.py index 3b9c02aec527..b0333d480c34 100644 --- a/airbyte-integrations/connectors/source-greenhouse/setup.py +++ b/airbyte-integrations/connectors/source-greenhouse/setup.py @@ -23,7 +23,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=["airbyte-cdk>=0.44.1", "dataclasses-jsonschema==2.15.1"], - package_data={"": ["*.json", "*.yaml", "schemas/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-harvest/setup.py b/airbyte-integrations/connectors/source-harvest/setup.py index 4c73f9283e52..d25d065699df 100644 --- a/airbyte-integrations/connectors/source-harvest/setup.py +++ b/airbyte-integrations/connectors/source-harvest/setup.py @@ -27,7 +27,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-hubspot/setup.py b/airbyte-integrations/connectors/source-hubspot/setup.py index a1831948c262..0f2f94491326 100644 --- a/airbyte-integrations/connectors/source-hubspot/setup.py +++ b/airbyte-integrations/connectors/source-hubspot/setup.py @@ -28,7 +28,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-instagram/setup.py b/airbyte-integrations/connectors/source-instagram/setup.py index cfaf2e20122f..b67ee6f39677 100644 --- a/airbyte-integrations/connectors/source-instagram/setup.py +++ b/airbyte-integrations/connectors/source-instagram/setup.py @@ -30,7 +30,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-iterable/setup.py b/airbyte-integrations/connectors/source-iterable/setup.py index fd2061fb89fb..8cdf6e07b666 100644 --- a/airbyte-integrations/connectors/source-iterable/setup.py +++ b/airbyte-integrations/connectors/source-iterable/setup.py @@ -30,5 +30,17 @@ extras_require={ "tests": TEST_REQUIREMENTS, }, - package_data={"": ["*.json", "schemas/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, ) diff --git a/airbyte-integrations/connectors/source-jira/setup.py b/airbyte-integrations/connectors/source-jira/setup.py index ccce70beddfc..653bc9e80640 100644 --- a/airbyte-integrations/connectors/source-jira/setup.py +++ b/airbyte-integrations/connectors/source-jira/setup.py @@ -26,7 +26,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-klaviyo/setup.py b/airbyte-integrations/connectors/source-klaviyo/setup.py index d9ac3a2fb807..854e9a61dac3 100644 --- a/airbyte-integrations/connectors/source-klaviyo/setup.py +++ b/airbyte-integrations/connectors/source-klaviyo/setup.py @@ -21,7 +21,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-kustomer-singer/setup.py b/airbyte-integrations/connectors/source-kustomer-singer/setup.py index dbaf47851ce7..bc8c57120807 100644 --- a/airbyte-integrations/connectors/source-kustomer-singer/setup.py +++ b/airbyte-integrations/connectors/source-kustomer-singer/setup.py @@ -69,7 +69,19 @@ def run(self): "develop": CustomDevelopCommand, "egg_info": CustomEggInfoCommand, }, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-kyriba/setup.py b/airbyte-integrations/connectors/source-kyriba/setup.py index 00c6c13f5657..fe8f94c888d0 100644 --- a/airbyte-integrations/connectors/source-kyriba/setup.py +++ b/airbyte-integrations/connectors/source-kyriba/setup.py @@ -27,7 +27,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-kyve/setup.py b/airbyte-integrations/connectors/source-kyve/setup.py index 128e97f87d25..709638a8574a 100644 --- a/airbyte-integrations/connectors/source-kyve/setup.py +++ b/airbyte-integrations/connectors/source-kyve/setup.py @@ -27,7 +27,19 @@ author_email="security@kyve.network", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-lever-hiring/setup.py b/airbyte-integrations/connectors/source-lever-hiring/setup.py index 0004a96f9d05..b87f0ba2038a 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/setup.py +++ b/airbyte-integrations/connectors/source-lever-hiring/setup.py @@ -28,7 +28,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-linkedin-ads/setup.py b/airbyte-integrations/connectors/source-linkedin-ads/setup.py index ceff2ed3bf72..c8ea4f0b34ca 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/setup.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/setup.py @@ -25,7 +25,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-linkedin-pages/setup.py b/airbyte-integrations/connectors/source-linkedin-pages/setup.py index 6f3d0697a13d..1b491a1f3e95 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/setup.py +++ b/airbyte-integrations/connectors/source-linkedin-pages/setup.py @@ -28,7 +28,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-linnworks/setup.py b/airbyte-integrations/connectors/source-linnworks/setup.py index 1256710cb3ff..54eed575b5d4 100644 --- a/airbyte-integrations/connectors/source-linnworks/setup.py +++ b/airbyte-integrations/connectors/source-linnworks/setup.py @@ -27,7 +27,19 @@ author_email="jv@labanoras.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-looker/setup.py b/airbyte-integrations/connectors/source-looker/setup.py index dc5e3295ad40..311a876d7fd9 100644 --- a/airbyte-integrations/connectors/source-looker/setup.py +++ b/airbyte-integrations/connectors/source-looker/setup.py @@ -31,7 +31,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-mailchimp/setup.py b/airbyte-integrations/connectors/source-mailchimp/setup.py index 0773da084484..58acdf82195d 100644 --- a/airbyte-integrations/connectors/source-mailchimp/setup.py +++ b/airbyte-integrations/connectors/source-mailchimp/setup.py @@ -23,6 +23,18 @@ "airbyte-cdk", "pytest~=6.1", ], - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={"tests": TEST_REQUIREMENTS}, ) diff --git a/airbyte-integrations/connectors/source-marketo/setup.py b/airbyte-integrations/connectors/source-marketo/setup.py index b8dfcde912ad..d1cccd943541 100644 --- a/airbyte-integrations/connectors/source-marketo/setup.py +++ b/airbyte-integrations/connectors/source-marketo/setup.py @@ -28,7 +28,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-microsoft-teams/setup.py b/airbyte-integrations/connectors/source-microsoft-teams/setup.py index f840cc27859a..07b984cad43d 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/setup.py +++ b/airbyte-integrations/connectors/source-microsoft-teams/setup.py @@ -30,7 +30,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-mixpanel/setup.py b/airbyte-integrations/connectors/source-mixpanel/setup.py index b89f8d01fbd7..9db4a5f2e5c1 100644 --- a/airbyte-integrations/connectors/source-mixpanel/setup.py +++ b/airbyte-integrations/connectors/source-mixpanel/setup.py @@ -23,7 +23,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-monday/setup.py b/airbyte-integrations/connectors/source-monday/setup.py index c99ddb8f0ed0..959f2bbcea68 100644 --- a/airbyte-integrations/connectors/source-monday/setup.py +++ b/airbyte-integrations/connectors/source-monday/setup.py @@ -27,7 +27,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-my-hours/setup.py b/airbyte-integrations/connectors/source-my-hours/setup.py index 4e8a87ce39ed..0f1e5c67af1a 100644 --- a/airbyte-integrations/connectors/source-my-hours/setup.py +++ b/airbyte-integrations/connectors/source-my-hours/setup.py @@ -28,7 +28,19 @@ author_email="wisse@vrowl.nl", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-notion/setup.py b/airbyte-integrations/connectors/source-notion/setup.py index d31dfbbdeee3..5ab3bec50ffd 100644 --- a/airbyte-integrations/connectors/source-notion/setup.py +++ b/airbyte-integrations/connectors/source-notion/setup.py @@ -29,7 +29,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-okta/setup.py b/airbyte-integrations/connectors/source-okta/setup.py index 1a15dc58fd47..1d2c69920b88 100644 --- a/airbyte-integrations/connectors/source-okta/setup.py +++ b/airbyte-integrations/connectors/source-okta/setup.py @@ -28,7 +28,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-orb/setup.py b/airbyte-integrations/connectors/source-orb/setup.py index 68f05af00a2c..da07d9e21ecd 100644 --- a/airbyte-integrations/connectors/source-orb/setup.py +++ b/airbyte-integrations/connectors/source-orb/setup.py @@ -21,7 +21,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-outreach/setup.py b/airbyte-integrations/connectors/source-outreach/setup.py index dc15e76330f8..87c10aedcb88 100644 --- a/airbyte-integrations/connectors/source-outreach/setup.py +++ b/airbyte-integrations/connectors/source-outreach/setup.py @@ -27,7 +27,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pardot/setup.py b/airbyte-integrations/connectors/source-pardot/setup.py index 47cbea1de2c0..da875f7e2057 100644 --- a/airbyte-integrations/connectors/source-pardot/setup.py +++ b/airbyte-integrations/connectors/source-pardot/setup.py @@ -27,7 +27,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-paystack/setup.py b/airbyte-integrations/connectors/source-paystack/setup.py index 22f2c755d169..8d37937dc667 100644 --- a/airbyte-integrations/connectors/source-paystack/setup.py +++ b/airbyte-integrations/connectors/source-paystack/setup.py @@ -23,7 +23,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pinterest/setup.py b/airbyte-integrations/connectors/source-pinterest/setup.py index d09828b5c17e..e75f6a651d01 100644 --- a/airbyte-integrations/connectors/source-pinterest/setup.py +++ b/airbyte-integrations/connectors/source-pinterest/setup.py @@ -26,7 +26,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/setup.py b/airbyte-integrations/connectors/source-pivotal-tracker/setup.py index e6c5edb752f1..050e33c2a4f7 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/setup.py +++ b/airbyte-integrations/connectors/source-pivotal-tracker/setup.py @@ -28,7 +28,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-posthog/setup.py b/airbyte-integrations/connectors/source-posthog/setup.py index 2f23e21f1cf8..5ebf94b4b079 100644 --- a/airbyte-integrations/connectors/source-posthog/setup.py +++ b/airbyte-integrations/connectors/source-posthog/setup.py @@ -25,6 +25,18 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={"tests": TEST_REQUIREMENTS}, ) diff --git a/airbyte-integrations/connectors/source-python-http-tutorial/setup.py b/airbyte-integrations/connectors/source-python-http-tutorial/setup.py index be40ba5ca6c6..35164f2108aa 100644 --- a/airbyte-integrations/connectors/source-python-http-tutorial/setup.py +++ b/airbyte-integrations/connectors/source-python-http-tutorial/setup.py @@ -16,5 +16,17 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=["airbyte-cdk", "pytest"], - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, ) diff --git a/airbyte-integrations/connectors/source-recharge/setup.py b/airbyte-integrations/connectors/source-recharge/setup.py index d51c828ffa5b..3e80432c9c57 100644 --- a/airbyte-integrations/connectors/source-recharge/setup.py +++ b/airbyte-integrations/connectors/source-recharge/setup.py @@ -27,7 +27,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-recurly/setup.py b/airbyte-integrations/connectors/source-recurly/setup.py index 16e2364aeb01..1d1cce5a7cd7 100644 --- a/airbyte-integrations/connectors/source-recurly/setup.py +++ b/airbyte-integrations/connectors/source-recurly/setup.py @@ -25,7 +25,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-rki-covid/setup.py b/airbyte-integrations/connectors/source-rki-covid/setup.py index 78722548d0b1..382e0f4e84b4 100644 --- a/airbyte-integrations/connectors/source-rki-covid/setup.py +++ b/airbyte-integrations/connectors/source-rki-covid/setup.py @@ -23,7 +23,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-s3/setup.py b/airbyte-integrations/connectors/source-s3/setup.py index cd2c48e2924d..aa86f2a7c246 100644 --- a/airbyte-integrations/connectors/source-s3/setup.py +++ b/airbyte-integrations/connectors/source-s3/setup.py @@ -23,7 +23,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-salesloft/setup.py b/airbyte-integrations/connectors/source-salesloft/setup.py index 58bcf6176dd6..f272bb4432de 100644 --- a/airbyte-integrations/connectors/source-salesloft/setup.py +++ b/airbyte-integrations/connectors/source-salesloft/setup.py @@ -25,7 +25,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/setup.py b/airbyte-integrations/connectors/source-scaffold-source-python/setup.py index d4b3ebe7601b..ebeda07f6998 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/setup.py +++ b/airbyte-integrations/connectors/source-scaffold-source-python/setup.py @@ -28,7 +28,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-search-metrics/setup.py b/airbyte-integrations/connectors/source-search-metrics/setup.py index dfc33d408c22..2e3f6e81f310 100644 --- a/airbyte-integrations/connectors/source-search-metrics/setup.py +++ b/airbyte-integrations/connectors/source-search-metrics/setup.py @@ -27,7 +27,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-sendgrid/setup.py b/airbyte-integrations/connectors/source-sendgrid/setup.py index 0bfa4f9dd3b2..6d6e3b920105 100644 --- a/airbyte-integrations/connectors/source-sendgrid/setup.py +++ b/airbyte-integrations/connectors/source-sendgrid/setup.py @@ -25,7 +25,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-sentry/setup.py b/airbyte-integrations/connectors/source-sentry/setup.py index d1677c524a4d..b434d911ec50 100644 --- a/airbyte-integrations/connectors/source-sentry/setup.py +++ b/airbyte-integrations/connectors/source-sentry/setup.py @@ -28,7 +28,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-sftp-bulk/setup.py b/airbyte-integrations/connectors/source-sftp-bulk/setup.py index e09d478f9007..282a57ca1fac 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/setup.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/setup.py @@ -27,7 +27,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-shopify/setup.py b/airbyte-integrations/connectors/source-shopify/setup.py index 7c87ac4cf564..dc38aaea43d1 100644 --- a/airbyte-integrations/connectors/source-shopify/setup.py +++ b/airbyte-integrations/connectors/source-shopify/setup.py @@ -20,7 +20,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-slack/setup.py b/airbyte-integrations/connectors/source-slack/setup.py index f1040f3acca2..1acab113b35d 100644 --- a/airbyte-integrations/connectors/source-slack/setup.py +++ b/airbyte-integrations/connectors/source-slack/setup.py @@ -23,7 +23,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=["airbyte-cdk", "pendulum>=2,<3"], - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-smartsheets/setup.py b/airbyte-integrations/connectors/source-smartsheets/setup.py index 661a68ca12be..006b0fab9b90 100644 --- a/airbyte-integrations/connectors/source-smartsheets/setup.py +++ b/airbyte-integrations/connectors/source-smartsheets/setup.py @@ -23,5 +23,17 @@ extras_require={ "tests": TEST_REQUIREMENTS, }, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, ) diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/setup.py b/airbyte-integrations/connectors/source-snapchat-marketing/setup.py index 71d4210cbe54..083a611ccab0 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/setup.py +++ b/airbyte-integrations/connectors/source-snapchat-marketing/setup.py @@ -23,7 +23,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-surveymonkey/setup.py b/airbyte-integrations/connectors/source-surveymonkey/setup.py index a72a0108a10b..a39c6d22bc80 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/setup.py +++ b/airbyte-integrations/connectors/source-surveymonkey/setup.py @@ -21,7 +21,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-talkdesk-explore/setup.py b/airbyte-integrations/connectors/source-talkdesk-explore/setup.py index 5df2ba2853d0..1ec623cfb4d5 100644 --- a/airbyte-integrations/connectors/source-talkdesk-explore/setup.py +++ b/airbyte-integrations/connectors/source-talkdesk-explore/setup.py @@ -27,7 +27,19 @@ author_email="alexandre.martins@saltpay.co", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-tempo/setup.py b/airbyte-integrations/connectors/source-tempo/setup.py index 3e2668d10d6c..37cf227b0f5b 100644 --- a/airbyte-integrations/connectors/source-tempo/setup.py +++ b/airbyte-integrations/connectors/source-tempo/setup.py @@ -27,7 +27,19 @@ author_email="thomas@gcompany.nl", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/setup.py b/airbyte-integrations/connectors/source-tiktok-marketing/setup.py index 21896188c6be..68b52709f03b 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/setup.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/setup.py @@ -21,6 +21,18 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={"tests": TEST_REQUIREMENTS}, ) diff --git a/airbyte-integrations/connectors/source-tplcentral/setup.py b/airbyte-integrations/connectors/source-tplcentral/setup.py index 45cd0ff8a633..b3d6343e0d1c 100644 --- a/airbyte-integrations/connectors/source-tplcentral/setup.py +++ b/airbyte-integrations/connectors/source-tplcentral/setup.py @@ -28,7 +28,19 @@ author_email="jv@labanoras.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-twilio/setup.py b/airbyte-integrations/connectors/source-twilio/setup.py index 19717cccc42f..d4124f980af5 100644 --- a/airbyte-integrations/connectors/source-twilio/setup.py +++ b/airbyte-integrations/connectors/source-twilio/setup.py @@ -25,7 +25,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-us-census/setup.py b/airbyte-integrations/connectors/source-us-census/setup.py index bf3858455362..a88a817b95a9 100644 --- a/airbyte-integrations/connectors/source-us-census/setup.py +++ b/airbyte-integrations/connectors/source-us-census/setup.py @@ -28,7 +28,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-youtube-analytics/setup.py b/airbyte-integrations/connectors/source-youtube-analytics/setup.py index 9b17c4365ad8..1a4c92407af1 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/setup.py +++ b/airbyte-integrations/connectors/source-youtube-analytics/setup.py @@ -27,7 +27,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-zendesk-chat/setup.py b/airbyte-integrations/connectors/source-zendesk-chat/setup.py index 8e1732196deb..88422f67917c 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/setup.py @@ -16,7 +16,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-zendesk-support/setup.py b/airbyte-integrations/connectors/source-zendesk-support/setup.py index e0466040c015..bddcdcef4208 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-support/setup.py @@ -22,7 +22,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-zendesk-talk/setup.py b/airbyte-integrations/connectors/source-zendesk-talk/setup.py index 204a1c5cded5..2c66cbc2c867 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-talk/setup.py @@ -26,7 +26,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-zenloop/setup.py b/airbyte-integrations/connectors/source-zenloop/setup.py index f6920042865b..8bc3ceffdfcb 100644 --- a/airbyte-integrations/connectors/source-zenloop/setup.py +++ b/airbyte-integrations/connectors/source-zenloop/setup.py @@ -28,7 +28,19 @@ author_email="alexander.batoulis@hometogo.com", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-zoho-crm/setup.py b/airbyte-integrations/connectors/source-zoho-crm/setup.py index a44e9b0529a7..15425f380be4 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/setup.py +++ b/airbyte-integrations/connectors/source-zoho-crm/setup.py @@ -27,7 +27,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-zuora/setup.py b/airbyte-integrations/connectors/source-zuora/setup.py index 43b397e5c183..6cec429e1996 100644 --- a/airbyte-integrations/connectors/source-zuora/setup.py +++ b/airbyte-integrations/connectors/source-zuora/setup.py @@ -27,7 +27,19 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, extras_require={ "tests": TEST_REQUIREMENTS, }, From 70b05a687e41466b8c80af6b5dd697ce80f52dd8 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 30 Jan 2024 12:26:28 +0100 Subject: [PATCH 17/96] Publish pokeapi to pypi (#34650) --- airbyte-integrations/connectors/source-pokeapi/metadata.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml index 076a75a780a4..2f6a410ea49b 100644 --- a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml +++ b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - "*" + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pokeapi registries: oss: enabled: true From b5e268e2a7ed6c8ad27f4700b97a3669e99eb2f5 Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk <35109939+tolik0@users.noreply.github.com> Date: Tue, 30 Jan 2024 13:40:17 +0200 Subject: [PATCH 18/96] :bug: Source Microsoft OneDrive: Fix Oauth (#34478) --- .../connectors/source-microsoft-onedrive/metadata.yaml | 2 +- .../source_microsoft_onedrive/source.py | 5 +++++ docs/integrations/sources/microsoft-onedrive.md | 5 +++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml b/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml index 0bab747ccf4d..f1b3317b475c 100644 --- a/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml @@ -16,7 +16,7 @@ data: connectorSubtype: api connectorType: source definitionId: 01d1c685-fd4a-4837-8f4c-93fe5a0d2188 - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.3 dockerRepository: airbyte/source-microsoft-onedrive githubIssueLabel: source-microsoft-onedrive icon: microsoft-onedrive.svg diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/source.py b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/source.py index e58d75d8625c..87cf25732fd8 100644 --- a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/source.py +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/source.py @@ -50,6 +50,11 @@ def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]}, }, }, + oauth_user_input_from_connector_config_specification={ + "type": "object", + "additionalProperties": False, + "properties": {"tenant_id": {"type": "string", "path_in_connector_config": ["credentials", "tenant_id"]}}, + }, ), ), ) diff --git a/docs/integrations/sources/microsoft-onedrive.md b/docs/integrations/sources/microsoft-onedrive.md index e3f1335d2571..6446c72bcb3b 100644 --- a/docs/integrations/sources/microsoft-onedrive.md +++ b/docs/integrations/sources/microsoft-onedrive.md @@ -121,6 +121,7 @@ The connector is restricted by normal Microsoft Graph [requests limitation](http | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------| -| 0.1.2 | 2021-12-22 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Add ql and sl to metadata | -| 0.1.1 | 2021-12-15 | [33539](https://github.com/airbytehq/airbyte/pull/33539) | Fix for docs name | +| 0.1.3 | 2024-01-24 | [34478](https://github.com/airbytehq/airbyte/pull/34478) | Fix OAuth | +| 0.1.2 | 2021-12-22 | [33745](https://github.com/airbytehq/airbyte/pull/33745) | Add ql and sl to metadata | +| 0.1.1 | 2021-12-15 | [33758](https://github.com/airbytehq/airbyte/pull/33758) | Fix for docs name | | 0.1.0 | 2021-12-06 | [32655](https://github.com/airbytehq/airbyte/pull/32655) | New source | From d7ddc776bd3cf8bf156db74458dc17b561bd03f5 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 30 Jan 2024 14:07:04 +0100 Subject: [PATCH 19/96] Publish to pypi (#34652) --- .../connectors/source-activecampaign/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-adjust/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-aha/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-aircall/metadata.yaml | 4 ++++ .../connectors/source-airtable/metadata.yaml | 4 ++++ .../connectors/source-alpha-vantage/metadata.yaml | 4 ++++ .../connectors/source-amazon-ads/metadata.yaml | 4 ++++ .../connectors/source-amazon-seller-partner/metadata.yaml | 4 ++++ .../connectors/source-amazon-sqs/metadata.yaml | 4 ++++ .../connectors/source-amplitude/metadata.yaml | 4 ++++ .../connectors/source-appfollow/metadata.yaml | 4 ++++ .../connectors/source-apple-search-ads/metadata.yaml | 4 ++++ .../connectors/source-appsflyer/metadata.yaml | 4 ++++ .../connectors/source-appstore-singer/metadata.yaml | 5 +++++ airbyte-integrations/connectors/source-asana/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-ashby/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-auth0/metadata.yaml | 4 ++++ .../connectors/source-aws-cloudtrail/metadata.yaml | 4 ++++ .../connectors/source-azure-blob-storage/metadata.yaml | 5 +++++ .../connectors/source-azure-table/metadata.yaml | 4 ++++ 20 files changed, 82 insertions(+) diff --git a/airbyte-integrations/connectors/source-activecampaign/metadata.yaml b/airbyte-integrations/connectors/source-activecampaign/metadata.yaml index 7b302f920aa0..e7b926edc0a0 100644 --- a/airbyte-integrations/connectors/source-activecampaign/metadata.yaml +++ b/airbyte-integrations/connectors/source-activecampaign/metadata.yaml @@ -12,6 +12,10 @@ data: icon: activecampaign.svg license: MIT name: ActiveCampaign + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-activecampaign registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-adjust/metadata.yaml b/airbyte-integrations/connectors/source-adjust/metadata.yaml index f3a7d7bb290a..f45ead0f39c6 100644 --- a/airbyte-integrations/connectors/source-adjust/metadata.yaml +++ b/airbyte-integrations/connectors/source-adjust/metadata.yaml @@ -12,6 +12,10 @@ data: icon: adjust.svg license: MIT name: Adjust + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-adjust registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-aha/metadata.yaml b/airbyte-integrations/connectors/source-aha/metadata.yaml index cbbecab8c080..88d029de3b5d 100644 --- a/airbyte-integrations/connectors/source-aha/metadata.yaml +++ b/airbyte-integrations/connectors/source-aha/metadata.yaml @@ -12,6 +12,10 @@ data: icon: aha.svg license: MIT name: Aha + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-aha registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-aircall/metadata.yaml b/airbyte-integrations/connectors/source-aircall/metadata.yaml index 1883e76b2e69..c76243302b10 100644 --- a/airbyte-integrations/connectors/source-aircall/metadata.yaml +++ b/airbyte-integrations/connectors/source-aircall/metadata.yaml @@ -8,6 +8,10 @@ data: icon: aircall.svg license: MIT name: Aircall + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-aircall registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-airtable/metadata.yaml b/airbyte-integrations/connectors/source-airtable/metadata.yaml index c722087101dd..8b3ed52a05ed 100644 --- a/airbyte-integrations/connectors/source-airtable/metadata.yaml +++ b/airbyte-integrations/connectors/source-airtable/metadata.yaml @@ -18,6 +18,10 @@ data: icon: airtable.svg license: MIT name: Airtable + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-airtable registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-alpha-vantage/metadata.yaml b/airbyte-integrations/connectors/source-alpha-vantage/metadata.yaml index d4f4a1a63e0f..2eb0807b3957 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/metadata.yaml +++ b/airbyte-integrations/connectors/source-alpha-vantage/metadata.yaml @@ -8,6 +8,10 @@ data: icon: alpha-vantage.svg license: MIT name: Alpha Vantage + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-alpha-vantage registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml b/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml index 0efe4b2b434d..1884fb64ce97 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml @@ -20,6 +20,10 @@ data: icon: amazonads.svg license: MIT name: Amazon Ads + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-amazon-ads registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml index 8c486641bdb4..4d6e4c510c45 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml @@ -22,6 +22,10 @@ data: icon: amazonsellerpartner.svg license: MIT name: Amazon Seller Partner + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-amazon-seller-partner registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-amazon-sqs/metadata.yaml b/airbyte-integrations/connectors/source-amazon-sqs/metadata.yaml index 457e4edafe0e..6b6bd34f1cfb 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-sqs/metadata.yaml @@ -12,6 +12,10 @@ data: icon: awssqs.svg license: MIT name: Amazon SQS + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-amazon-sqs registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-amplitude/metadata.yaml b/airbyte-integrations/connectors/source-amplitude/metadata.yaml index fac740931651..1a51b2ef99c2 100644 --- a/airbyte-integrations/connectors/source-amplitude/metadata.yaml +++ b/airbyte-integrations/connectors/source-amplitude/metadata.yaml @@ -18,6 +18,10 @@ data: icon: amplitude.svg license: MIT name: Amplitude + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-amplitude registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-appfollow/metadata.yaml b/airbyte-integrations/connectors/source-appfollow/metadata.yaml index 7e9cb43f0cac..ad5e49e07ca4 100644 --- a/airbyte-integrations/connectors/source-appfollow/metadata.yaml +++ b/airbyte-integrations/connectors/source-appfollow/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - https://api.appfollow.io + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-appfollow registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-apple-search-ads/metadata.yaml b/airbyte-integrations/connectors/source-apple-search-ads/metadata.yaml index 1406f248390e..bbbaecfeef94 100644 --- a/airbyte-integrations/connectors/source-apple-search-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-apple-search-ads/metadata.yaml @@ -8,6 +8,10 @@ data: icon: apple.svg license: MIT name: Apple Search Ads + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-apple-search-ads registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-appsflyer/metadata.yaml b/airbyte-integrations/connectors/source-appsflyer/metadata.yaml index 4f3c263c6cbb..21d47e8efcbd 100644 --- a/airbyte-integrations/connectors/source-appsflyer/metadata.yaml +++ b/airbyte-integrations/connectors/source-appsflyer/metadata.yaml @@ -8,6 +8,10 @@ data: icon: appsflyer.svg license: MIT name: AppsFlyer + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-appsflyer registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-appstore-singer/metadata.yaml b/airbyte-integrations/connectors/source-appstore-singer/metadata.yaml index cb0d55be57b3..703a0000210f 100644 --- a/airbyte-integrations/connectors/source-appstore-singer/metadata.yaml +++ b/airbyte-integrations/connectors/source-appstore-singer/metadata.yaml @@ -8,6 +8,11 @@ data: icon: appstore.svg license: MIT name: Appstore + remoteRegistries: + pypi: + enabled: false + # TODO: Set enabled=true after `airbyte-lib-validate-source` is passing. + packageName: airbyte-source-appstore-singer registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-asana/metadata.yaml b/airbyte-integrations/connectors/source-asana/metadata.yaml index d3a4dd21bda3..ae99033996f8 100644 --- a/airbyte-integrations/connectors/source-asana/metadata.yaml +++ b/airbyte-integrations/connectors/source-asana/metadata.yaml @@ -15,6 +15,10 @@ data: icon: asana.svg license: MIT name: Asana + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-asana registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-ashby/metadata.yaml b/airbyte-integrations/connectors/source-ashby/metadata.yaml index 510c05376240..94a4355863a3 100644 --- a/airbyte-integrations/connectors/source-ashby/metadata.yaml +++ b/airbyte-integrations/connectors/source-ashby/metadata.yaml @@ -8,6 +8,10 @@ data: icon: ashby.svg license: MIT name: Ashby + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-ashby registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-auth0/metadata.yaml b/airbyte-integrations/connectors/source-auth0/metadata.yaml index 9566ee9d7800..21016c9467fd 100644 --- a/airbyte-integrations/connectors/source-auth0/metadata.yaml +++ b/airbyte-integrations/connectors/source-auth0/metadata.yaml @@ -17,6 +17,10 @@ data: icon: auth0.svg license: MIT name: Auth0 + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-auth0 registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/metadata.yaml b/airbyte-integrations/connectors/source-aws-cloudtrail/metadata.yaml index 3b3240c49425..f4d483e04c68 100644 --- a/airbyte-integrations/connectors/source-aws-cloudtrail/metadata.yaml +++ b/airbyte-integrations/connectors/source-aws-cloudtrail/metadata.yaml @@ -12,6 +12,10 @@ data: icon: awscloudtrail.svg license: MIT name: AWS CloudTrail + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-aws-cloudtrail registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml b/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml index 839510e2e159..14923e9a8c79 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml +++ b/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml @@ -14,6 +14,11 @@ data: icon: azureblobstorage.svg license: MIT name: Azure Blob Storage + remoteRegistries: + pypi: + enabled: false + # TODO: Set enabled=true after `airbyte-lib-validate-source` is passing. + packageName: airbyte-source-azure-blob-storage registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-azure-table/metadata.yaml b/airbyte-integrations/connectors/source-azure-table/metadata.yaml index 06efb503b20c..6fa88db58124 100644 --- a/airbyte-integrations/connectors/source-azure-table/metadata.yaml +++ b/airbyte-integrations/connectors/source-azure-table/metadata.yaml @@ -8,6 +8,10 @@ data: icon: azureblobstorage.svg license: MIT name: Azure Table Storage + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-azure-table registries: cloud: enabled: true From 8cef86901b98a3947ed4ef681b90d0cdcbf58504 Mon Sep 17 00:00:00 2001 From: Maxime Carbonneau-Leclerc <3360483+maxi297@users.noreply.github.com> Date: Tue, 30 Jan 2024 08:45:49 -0500 Subject: [PATCH 20/96] Emit state when no partitions are generated for ccdk (#34605) --- .../concurrent_read_processor.py | 14 +++++++------- .../streams/concurrent/abstract_stream.py | 8 ++++++++ .../sources/streams/concurrent/adapters.py | 1 + .../sources/streams/concurrent/cursor.py | 18 ++++++++++++++++++ .../streams/concurrent/default_stream.py | 7 +++++++ .../scenarios/incremental_scenarios.py | 3 +++ .../scenarios/stream_facade_scenarios.py | 2 ++ ...thread_based_concurrent_stream_scenarios.py | 8 ++++++++ .../test_concurrent_read_processor.py | 17 ++++++++++++----- .../streams/concurrent/test_default_stream.py | 7 ++++++- .../concurrent/test_partition_enqueuer.py | 2 +- .../sources/test_concurrent_source.py | 5 +++++ 12 files changed, 78 insertions(+), 14 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py index acfc0c039694..24ac315c526e 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py @@ -66,14 +66,12 @@ def on_partition_generation_completed(self, sentinel: PartitionGenerationComplet """ stream_name = sentinel.stream.name self._streams_currently_generating_partitions.remove(sentinel.stream.name) - ret = [] # It is possible for the stream to already be done if no partitions were generated # If the partition generation process was completed and there are no partitions left to process, the stream is done if self._is_stream_done(stream_name) or len(self._streams_to_running_partitions[stream_name]) == 0: - ret.append(self._on_stream_is_done(stream_name)) + yield from self._on_stream_is_done(stream_name) if self._stream_instances_to_start_partition_generation: - ret.append(self.start_next_partition_generator()) - return ret + yield self.start_next_partition_generator() def on_partition(self, partition: Partition) -> None: """ @@ -102,7 +100,7 @@ def on_partition_complete_sentinel(self, sentinel: PartitionCompleteSentinel) -> partitions_running.remove(partition) # If all partitions were generated and this was the last one, the stream is done if partition.stream_name() not in self._streams_currently_generating_partitions and len(partitions_running) == 0: - yield self._on_stream_is_done(partition.stream_name()) + yield from self._on_stream_is_done(partition.stream_name()) yield from self._message_repository.consume_queue() def on_record(self, record: Record) -> Iterable[AirbyteMessage]: @@ -171,13 +169,15 @@ def is_done(self) -> bool: def _is_stream_done(self, stream_name: str) -> bool: return stream_name in self._streams_done - def _on_stream_is_done(self, stream_name: str) -> AirbyteMessage: + def _on_stream_is_done(self, stream_name: str) -> Iterable[AirbyteMessage]: self._logger.info(f"Read {self._record_counter[stream_name]} records from {stream_name} stream") self._logger.info(f"Marking stream {stream_name} as STOPPED") stream = self._stream_name_to_instance[stream_name] + stream.cursor.ensure_at_least_one_state_emitted() + yield from self._message_repository.consume_queue() self._logger.info(f"Finished syncing {stream.name}") self._streams_done.add(stream_name) - return stream_status_as_airbyte_message(stream.as_airbyte_stream(), AirbyteStreamStatus.COMPLETE) + yield stream_status_as_airbyte_message(stream.as_airbyte_stream(), AirbyteStreamStatus.COMPLETE) def _stop_streams(self) -> Iterable[AirbyteMessage]: self._thread_pool_manager.shutdown() diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/abstract_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/abstract_stream.py index d98e7a7b5498..d48c32c48dbe 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/abstract_stream.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/abstract_stream.py @@ -7,6 +7,7 @@ from airbyte_cdk.models import AirbyteStream from airbyte_cdk.sources.streams.concurrent.availability_strategy import StreamAvailability +from airbyte_cdk.sources.streams.concurrent.cursor import Cursor from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition from deprecated.classic import deprecated @@ -81,3 +82,10 @@ def log_stream_sync_configuration(self) -> None: """ Logs the stream's configuration for debugging purposes. """ + + @property + @abstractmethod + def cursor(self) -> Cursor: + """ + :return: The cursor associated with this stream. + """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/adapters.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/adapters.py index ba13a467238d..86542618354f 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/adapters.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/adapters.py @@ -89,6 +89,7 @@ def create_from_stream( primary_key=pk, cursor_field=cursor_field, logger=logger, + cursor=cursor, ), stream, cursor, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/cursor.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/cursor.py index 282498db1783..e63358b715d5 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/cursor.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/cursor.py @@ -56,6 +56,14 @@ def close_partition(self, partition: Partition) -> None: """ raise NotImplementedError() + @abstractmethod + def ensure_at_least_one_state_emitted(self) -> None: + """ + State messages are emitted when a partition is closed. However, the platform expects at least one state to be emitted per sync per + stream. Hence, if no partitions are generated, this method needs to be called. + """ + raise NotImplementedError() + class NoopCursor(Cursor): @property @@ -68,6 +76,9 @@ def observe(self, record: Record) -> None: def close_partition(self, partition: Partition) -> None: pass + def ensure_at_least_one_state_emitted(self) -> None: + pass + class ConcurrentCursor(Cursor): _START_BOUNDARY = 0 @@ -179,3 +190,10 @@ def _extract_from_slice(self, partition: Partition, key: str) -> Comparable: return self._connector_state_converter.parse_value(_slice[key]) # type: ignore # we expect the devs to specify a key that would return a Comparable except KeyError as exception: raise KeyError(f"Partition is expected to have key `{key}` but could not be found") from exception + + def ensure_at_least_one_state_emitted(self) -> None: + """ + The platform expect to have at least one state message on successful syncs. Hence, whatever happens, we expect this method to be + called. + """ + self._emit_state_message() diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/default_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/default_stream.py index 8606d273bb4f..3e839cb3959e 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/default_stream.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/default_stream.py @@ -9,6 +9,7 @@ from airbyte_cdk.models import AirbyteStream, SyncMode from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream from airbyte_cdk.sources.streams.concurrent.availability_strategy import AbstractAvailabilityStrategy, StreamAvailability +from airbyte_cdk.sources.streams.concurrent.cursor import Cursor, NoopCursor from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition from airbyte_cdk.sources.streams.concurrent.partitions.partition_generator import PartitionGenerator @@ -23,6 +24,7 @@ def __init__( primary_key: List[str], cursor_field: Optional[str], logger: Logger, + cursor: Optional[Cursor], namespace: Optional[str] = None, ) -> None: self._stream_partition_generator = partition_generator @@ -32,6 +34,7 @@ def __init__( self._primary_key = primary_key self._cursor_field = cursor_field self._logger = logger + self._cursor = cursor or NoopCursor() self._namespace = namespace def generate_partitions(self) -> Iterable[Partition]: @@ -77,3 +80,7 @@ def log_stream_sync_configuration(self) -> None: "cursor_field": self.cursor_field, }, ) + + @property + def cursor(self) -> Cursor: + return self._cursor diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py index 72a0425bc098..fd8c2ed9ac56 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py @@ -76,6 +76,7 @@ {"data": {"id": "3", "cursor_field": 2}, "stream": "stream1"}, {"data": {"id": "4", "cursor_field": 3}, "stream": "stream1"}, {"stream1": {"cursor_field": 2}}, + {"stream1": {"cursor_field": 2}}, # see Cursor.ensure_at_least_one_state_emitted ] ) .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"}) @@ -152,6 +153,7 @@ {"data": {"id": "3", "cursor_field": 2}, "stream": "stream1"}, {"data": {"id": "4", "cursor_field": 3}, "stream": "stream1"}, {"stream1": {"cursor_field": 2}}, + {"stream1": {"cursor_field": 2}}, # see Cursor.ensure_at_least_one_state_emitted ] ) .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"}) @@ -239,6 +241,7 @@ {"data": {"id": "3", "cursor_field": 2}, "stream": "stream1"}, {"data": {"id": "4", "cursor_field": 3}, "stream": "stream1"}, {"stream1": {"cursor_field": 2}}, + {"stream1": {"cursor_field": 2}}, # see Cursor.ensure_at_least_one_state_emitted ] ) .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"}) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py index ae66d3a44374..2080126a509e 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py @@ -361,6 +361,7 @@ {"data": {"id": "3", "cursor_field": 2}, "stream": "stream1"}, {"data": {"id": "4", "cursor_field": 3}, "stream": "stream1"}, {"stream1": {"cursor_field": 2}}, + {"stream1": {"cursor_field": 2}}, # see Cursor.ensure_at_least_one_state_emitted ] ) .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"}) @@ -403,6 +404,7 @@ {"data": {"id": "1", "cursor_field": 0}, "stream": "stream1"}, {"data": {"id": "2", "cursor_field": 3}, "stream": "stream1"}, {"stream1": {"cursor_field": 3}}, + {"stream1": {"cursor_field": 3}}, # see Cursor.ensure_at_least_one_state_emitted ] ) .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"}) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py index 2f4ab9b9fccb..e1eb81445d4a 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py @@ -4,6 +4,7 @@ import logging from airbyte_cdk.sources.message import InMemoryMessageRepository +from airbyte_cdk.sources.streams.concurrent.cursor import NoopCursor from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream from airbyte_cdk.sources.streams.concurrent.partitions.record import Record from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder @@ -29,6 +30,7 @@ primary_key=[], cursor_field=None, logger=logging.getLogger("test_logger"), + cursor=NoopCursor(), ) _id_only_stream_with_slice_logger = DefaultStream( @@ -46,6 +48,7 @@ primary_key=[], cursor_field=None, logger=logging.getLogger("test_logger"), + cursor=NoopCursor(), ) _id_only_stream_with_primary_key = DefaultStream( @@ -63,6 +66,7 @@ primary_key=["id"], cursor_field=None, logger=logging.getLogger("test_logger"), + cursor=NoopCursor(), ) _id_only_stream_multiple_partitions = DefaultStream( @@ -83,6 +87,7 @@ primary_key=[], cursor_field=None, logger=logging.getLogger("test_logger"), + cursor=NoopCursor(), ) _id_only_stream_multiple_partitions_concurrency_level_two = DefaultStream( @@ -103,6 +108,7 @@ primary_key=[], cursor_field=None, logger=logging.getLogger("test_logger"), + cursor=NoopCursor(), ) _stream_raising_exception = DefaultStream( @@ -120,6 +126,7 @@ primary_key=[], cursor_field=None, logger=logging.getLogger("test_logger"), + cursor=NoopCursor(), ) test_concurrent_cdk_single_stream = ( @@ -246,6 +253,7 @@ primary_key=[], cursor_field=None, logger=logging.getLogger("test_logger"), + cursor=NoopCursor(), ), ] ) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py index e33ce5b4df72..f03fe0c0eb43 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py @@ -3,7 +3,7 @@ # import logging import unittest -from unittest.mock import Mock +from unittest.mock import Mock, call import freezegun from airbyte_cdk.models import ( @@ -32,6 +32,7 @@ _STREAM_NAME = "stream" _ANOTHER_STREAM_NAME = "stream2" +_ANY_AIRBYTE_MESSAGE = Mock(spec=AirbyteMessage) class TestConcurrentReadProcessor(unittest.TestCase): @@ -110,6 +111,10 @@ def test_handle_partition_done_no_other_streams_to_generate_partitions_for(self) @freezegun.freeze_time("2020-01-01T00:00:00") def test_handle_last_stream_partition_done(self): + in_order_validation_mock = Mock() + in_order_validation_mock.attach_mock(self._another_stream, "_another_stream") + in_order_validation_mock.attach_mock(self._message_repository, '_message_repository') + self._message_repository.consume_queue.return_value = iter([_ANY_AIRBYTE_MESSAGE]) stream_instances_to_read_from = [self._another_stream] handler = ConcurrentReadProcessor( @@ -124,9 +129,10 @@ def test_handle_last_stream_partition_done(self): handler.start_next_partition_generator() sentinel = PartitionGenerationCompletedSentinel(self._another_stream) - messages = handler.on_partition_generation_completed(sentinel) + messages = list(handler.on_partition_generation_completed(sentinel)) expected_messages = [ + _ANY_AIRBYTE_MESSAGE, AirbyteMessage( type=MessageType.TRACE, trace=AirbyteTraceMessage( @@ -140,6 +146,7 @@ def test_handle_last_stream_partition_done(self): ) ] assert expected_messages == messages + assert in_order_validation_mock.mock_calls.index(call._another_stream.cursor.ensure_at_least_one_state_emitted) < in_order_validation_mock.mock_calls.index(call._message_repository.consume_queue) def test_handle_partition(self): stream_instances_to_read_from = [self._another_stream] @@ -236,7 +243,7 @@ def test_handle_on_partition_complete_sentinel_yields_status_message_if_the_stre ) handler.start_next_partition_generator() handler.on_partition(self._a_closed_partition) - handler.on_partition_generation_completed(PartitionGenerationCompletedSentinel(self._another_stream)) + list(handler.on_partition_generation_completed(PartitionGenerationCompletedSentinel(self._another_stream))) sentinel = PartitionCompleteSentinel(self._a_closed_partition) @@ -543,8 +550,8 @@ def test_on_exception_does_not_stop_streams_that_are_already_done(self): handler.start_next_partition_generator() handler.on_partition(self._an_open_partition) - handler.on_partition_generation_completed(PartitionGenerationCompletedSentinel(self._stream)) - handler.on_partition_generation_completed(PartitionGenerationCompletedSentinel(self._another_stream)) + list(handler.on_partition_generation_completed(PartitionGenerationCompletedSentinel(self._stream))) + list(handler.on_partition_generation_completed(PartitionGenerationCompletedSentinel(self._another_stream))) another_stream = Mock(spec=AbstractStream) another_stream.name = _STREAM_NAME diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_default_stream.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_default_stream.py index 818c2862bb8b..fb40368d98b3 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_default_stream.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_default_stream.py @@ -6,7 +6,7 @@ from airbyte_cdk.models import AirbyteStream, SyncMode from airbyte_cdk.sources.streams.concurrent.availability_strategy import STREAM_AVAILABLE -from airbyte_cdk.sources.streams.concurrent.cursor import Cursor +from airbyte_cdk.sources.streams.concurrent.cursor import Cursor, NoopCursor from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream @@ -28,6 +28,7 @@ def setUp(self): self._primary_key, self._cursor_field, self._logger, + NoopCursor(), ) def test_get_json_schema(self): @@ -88,6 +89,7 @@ def test_as_airbyte_stream_with_primary_key(self): ["id"], self._cursor_field, self._logger, + NoopCursor(), ) expected_airbyte_stream = AirbyteStream( @@ -119,6 +121,7 @@ def test_as_airbyte_stream_with_composite_primary_key(self): ["id_a", "id_b"], self._cursor_field, self._logger, + NoopCursor(), ) expected_airbyte_stream = AirbyteStream( @@ -150,6 +153,7 @@ def test_as_airbyte_stream_with_a_cursor(self): self._primary_key, "date", self._logger, + NoopCursor(), ) expected_airbyte_stream = AirbyteStream( @@ -174,6 +178,7 @@ def test_as_airbyte_stream_with_namespace(self): self._primary_key, self._cursor_field, self._logger, + NoopCursor(), namespace="test", ) expected_airbyte_stream = AirbyteStream( diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_enqueuer.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_enqueuer.py index c3ce277fb8c7..5b832adeaec0 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_enqueuer.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_enqueuer.py @@ -32,7 +32,7 @@ def test_given_no_partitions_when_generate_partitions_then_do_not_wait(self, moc assert mocked_sleep.call_count == 0 - def test_given_partitions_when_generate_partitions_then_only_push_sentinel(self): + def test_given_no_partitions_when_generate_partitions_then_only_push_sentinel(self): self._thread_pool_manager.prune_to_validate_has_reached_futures_limit.return_value = True stream = self._a_stream([]) diff --git a/airbyte-cdk/python/unit_tests/sources/test_concurrent_source.py b/airbyte-cdk/python/unit_tests/sources/test_concurrent_source.py index ca5c669a27c6..ebd082a2b152 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_concurrent_source.py +++ b/airbyte-cdk/python/unit_tests/sources/test_concurrent_source.py @@ -12,6 +12,7 @@ from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream from airbyte_cdk.sources.streams.concurrent.availability_strategy import StreamAvailability, StreamAvailable, StreamUnavailable +from airbyte_cdk.sources.streams.concurrent.cursor import Cursor, NoopCursor from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition from airbyte_cdk.sources.streams.concurrent.partitions.record import Record from airbyte_protocol.models import AirbyteStream @@ -72,6 +73,10 @@ def as_airbyte_stream(self) -> AirbyteStream: def log_stream_sync_configuration(self) -> None: raise NotImplementedError + @property + def cursor(self) -> Cursor: + return NoopCursor() + class _MockPartition(Partition): def __init__(self, name: str): From 8932636e399d85909d681fc57e5aec18440bdb80 Mon Sep 17 00:00:00 2001 From: Maxime Carbonneau-Leclerc <3360483+maxi297@users.noreply.github.com> Date: Tue, 30 Jan 2024 08:46:03 -0500 Subject: [PATCH 21/96] Have StateBuilder return our actual state object and not simply a dict (#34625) --- .../airbyte_cdk/test/entrypoint_wrapper.py | 15 +- .../python/airbyte_cdk/test/state_builder.py | 12 +- .../scenarios/incremental_scenarios.py | 179 +++++------------- .../scenarios/incremental_scenarios.py | 19 +- .../test/test_entrypoint_wrapper.py | 3 +- 5 files changed, 79 insertions(+), 149 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/test/entrypoint_wrapper.py b/airbyte-cdk/python/airbyte_cdk/test/entrypoint_wrapper.py index 06d5e0ebeb20..612b2742ea1e 100644 --- a/airbyte-cdk/python/airbyte_cdk/test/entrypoint_wrapper.py +++ b/airbyte-cdk/python/airbyte_cdk/test/entrypoint_wrapper.py @@ -26,7 +26,16 @@ from airbyte_cdk.exception_handler import assemble_uncaught_exception from airbyte_cdk.logger import AirbyteLogFormatter from airbyte_cdk.sources import Source -from airbyte_protocol.models import AirbyteLogMessage, AirbyteMessage, AirbyteStreamStatus, ConfiguredAirbyteCatalog, Level, TraceType, Type +from airbyte_protocol.models import ( + AirbyteLogMessage, + AirbyteMessage, + AirbyteStateMessage, + AirbyteStreamStatus, + ConfiguredAirbyteCatalog, + Level, + TraceType, + Type, +) from pydantic.error_wrappers import ValidationError @@ -104,7 +113,7 @@ def read( source: Source, config: Mapping[str, Any], catalog: ConfiguredAirbyteCatalog, - state: Optional[Any] = None, + state: Optional[List[AirbyteStateMessage]] = None, expecting_exception: bool = False, ) -> EntrypointOutput: """ @@ -133,7 +142,7 @@ def read( args.extend( [ "--state", - make_file(tmp_directory_path / "state.json", state), + make_file(tmp_directory_path / "state.json", f"[{','.join([stream_state.json() for stream_state in state])}]"), ] ) args.append("--debug") diff --git a/airbyte-cdk/python/airbyte_cdk/test/state_builder.py b/airbyte-cdk/python/airbyte_cdk/test/state_builder.py index 96c9a6161172..1c356afef889 100644 --- a/airbyte-cdk/python/airbyte_cdk/test/state_builder.py +++ b/airbyte-cdk/python/airbyte_cdk/test/state_builder.py @@ -1,14 +1,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. -from typing import Any, Dict, List +from typing import Any, List + +from airbyte_protocol.models import AirbyteStateMessage class StateBuilder: def __init__(self) -> None: - self._state: List[Dict[str, Any]] = [] + self._state: List[AirbyteStateMessage] = [] def with_stream_state(self, stream_name: str, state: Any) -> "StateBuilder": - self._state.append({ + self._state.append(AirbyteStateMessage.parse_obj({ "type": "STREAM", "stream": { "stream_state": state, @@ -16,8 +18,8 @@ def with_stream_state(self, stream_name: str, state: Any) -> "StateBuilder": "name": stream_name } } - }) + })) return self - def build(self) -> List[Dict[str, Any]]: + def build(self) -> List[AirbyteStateMessage]: return self._state diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/incremental_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/incremental_scenarios.py index 3c3195fbac61..6691ba6ba320 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/incremental_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/incremental_scenarios.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.test.state_builder import StateBuilder from unit_tests.sources.file_based.helpers import LowHistoryLimitCursor from unit_tests.sources.file_based.scenarios.file_based_source_builder import FileBasedSourceBuilder from unit_tests.sources.file_based.scenarios.scenario_builder import IncrementalScenarioConfig, TestScenarioBuilder @@ -39,17 +40,9 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[ - { - "type": "STREAM", - "stream": { - "stream_state": { - "history": {"some_old_file.csv": "2023-06-01T03:54:07.000000Z"}, - }, - "stream_descriptor": {"name": "stream1"}, - }, - } - ], + input_state=StateBuilder().with_stream_state("stream1", { + "history": {"some_old_file.csv": "2023-06-01T03:54:07.000000Z"}, + }).build(), ) ) .set_expected_records( @@ -140,17 +133,9 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[ - { - "type": "STREAM", - "stream": { - "stream_state": { - "history": {"a.csv": "2023-06-05T03:54:07.000000Z"}, - }, - "stream_descriptor": {"name": "stream1"}, - }, - } - ], + input_state=StateBuilder().with_stream_state("stream1", { + "history": {"a.csv": "2023-06-05T03:54:07.000000Z"}, + }).build(), ) ) .set_expected_records( @@ -223,17 +208,9 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[ - { - "type": "STREAM", - "stream": { - "stream_state": { - "history": {"a.csv": "2023-06-01T03:54:07.000000Z"}, - }, - "stream_descriptor": {"name": "stream1"}, - }, - } - ], + input_state=StateBuilder().with_stream_state("stream1", { + "history": {"a.csv": "2023-06-01T03:54:07.000000Z"}, + }).build(), ) ) .set_expected_records( @@ -377,7 +354,7 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[], + input_state=StateBuilder().build(), ) ) ).build() @@ -499,7 +476,7 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[], + input_state=StateBuilder().build(), ) ) ).build() @@ -593,15 +570,9 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[ - { - "type": "STREAM", - "stream": { - "stream_state": {"history": {"recent_file.csv": "2023-07-15T23:59:59.000000Z"}}, - "stream_descriptor": {"name": "stream1"}, - }, - } - ], + input_state=StateBuilder().with_stream_state("stream1", { + "history": {"recent_file.csv": "2023-07-15T23:59:59.000000Z"}, + }).build(), ) ) ).build() @@ -731,7 +702,7 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[], + input_state=StateBuilder().build(), ) ) ).build() @@ -891,7 +862,7 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[], + input_state=StateBuilder().build(), ) ) ).build() @@ -1035,15 +1006,9 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[ - { - "type": "STREAM", - "stream": { - "stream_state": {"history": {"a.csv": "2023-06-05T03:54:07.000000Z"}}, - "stream_descriptor": {"name": "stream1"}, - }, - } - ], + input_state=StateBuilder().with_stream_state("stream1", { + "history": {"a.csv": "2023-06-05T03:54:07.000000Z"}, + }).build(), ) ) ).build() @@ -1163,17 +1128,9 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[ - { - "type": "STREAM", - "stream": { - "stream_state": { - "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "c.csv": "2023-06-06T03:54:07.000000Z"}, - }, - "stream_descriptor": {"name": "stream1"}, - }, - } - ], + input_state=StateBuilder().with_stream_state("stream1", { + "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "c.csv": "2023-06-06T03:54:07.000000Z"}, + }).build(), ) ) ).build() @@ -1348,21 +1305,13 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[ - { - "type": "STREAM", - "stream": { - "stream_state": { - "history": { - "very_very_old_file.csv": "2023-06-01T03:54:07.000000Z", - "very_old_file.csv": "2023-06-02T03:54:07.000000Z", - "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z", - }, - }, - "stream_descriptor": {"name": "stream1"}, - }, - } - ], + input_state=StateBuilder().with_stream_state("stream1", { + "history": { + "very_very_old_file.csv": "2023-06-01T03:54:07.000000Z", + "very_old_file.csv": "2023-06-02T03:54:07.000000Z", + "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z", + }, + }).build(), ) ) ).build() @@ -1546,7 +1495,7 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[], + input_state=StateBuilder().build(), ) ) ).build() @@ -1652,21 +1601,13 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[ - { - "type": "STREAM", - "stream": { - "stream_state": { - "history": { - "b.csv": "2023-06-05T03:54:07.000000Z", - "c.csv": "2023-06-05T03:54:07.000000Z", - "d.csv": "2023-06-05T03:54:07.000000Z", - }, - }, - "stream_descriptor": {"name": "stream1"}, - }, - } - ], + input_state=StateBuilder().with_stream_state("stream1", { + "history": { + "b.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-05T03:54:07.000000Z", + "d.csv": "2023-06-05T03:54:07.000000Z", + }, + }).build(), ) ) ).build() @@ -1794,21 +1735,13 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[ - { - "type": "STREAM", - "stream": { - "stream_state": { - "history": { - "c.csv": "2023-06-07T03:54:07.000000Z", - "d.csv": "2023-06-08T03:54:07.000000Z", - "e.csv": "2023-06-08T03:54:07.000000Z", - }, - }, - "stream_descriptor": {"name": "stream1"}, - }, - } - ], + input_state=StateBuilder().with_stream_state("stream1", { + "history": { + "c.csv": "2023-06-07T03:54:07.000000Z", + "d.csv": "2023-06-08T03:54:07.000000Z", + "e.csv": "2023-06-08T03:54:07.000000Z", + }, + }).build(), ) ) ).build() @@ -1962,21 +1895,13 @@ ) .set_incremental_scenario_config( IncrementalScenarioConfig( - input_state=[ - { - "type": "STREAM", - "stream": { - "stream_state": { - "history": { - "old_file.csv": "2023-06-05T00:00:00.000000Z", - "c.csv": "2023-06-07T03:54:07.000000Z", - "d.csv": "2023-06-08T03:54:07.000000Z", - }, - }, - "stream_descriptor": {"name": "stream1"}, - }, - } - ], + input_state=StateBuilder().with_stream_state("stream1", { + "history": { + "old_file.csv": "2023-06-05T00:00:00.000000Z", + "c.csv": "2023-06-07T03:54:07.000000Z", + "d.csv": "2023-06-08T03:54:07.000000Z", + }, + }).build(), ) ) ).build() diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py index fd8c2ed9ac56..08df8db39b7f 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py @@ -3,6 +3,7 @@ # from airbyte_cdk.sources.streams.concurrent.cursor import CursorField from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_state_converter import ConcurrencyCompatibleStateType +from airbyte_cdk.test.state_builder import StateBuilder from unit_tests.sources.file_based.scenarios.scenario_builder import IncrementalScenarioConfig, TestScenarioBuilder from unit_tests.sources.streams.concurrent.scenarios.stream_facade_builder import StreamFacadeSourceBuilder from unit_tests.sources.streams.concurrent.scenarios.utils import MockStream @@ -85,7 +86,7 @@ ) -LEGACY_STATE = [{"type": "STREAM", "stream": {"stream_state": {"cursor_field": 0}, "stream_descriptor": {"name": "stream1"}}}] +LEGACY_STATE = StateBuilder().with_stream_state("stream1", {"cursor_field": 0}).build() test_incremental_stream_without_slice_boundaries_with_legacy_state = ( TestScenarioBuilder() .set_name("test_incremental_stream_without_slice_boundaries_with_legacy_state") @@ -162,18 +163,10 @@ ) -CONCURRENT_STATE = [ - { - "type": "STREAM", - "stream": { - "stream_state": { - "slices": [{"start": 0, "end": 0}], - "state_type": ConcurrencyCompatibleStateType.date_range.value, - }, - "stream_descriptor": {"name": "stream1"}, - }, - }, -] +CONCURRENT_STATE = StateBuilder().with_stream_state("stream1", { + "slices": [{"start": 0, "end": 0}], + "state_type": ConcurrencyCompatibleStateType.date_range.value, +}).build() test_incremental_stream_without_slice_boundaries_with_concurrent_state = ( TestScenarioBuilder() .set_name("test_incremental_stream_without_slice_boundaries_with_concurrent_state") diff --git a/airbyte-cdk/python/unit_tests/test/test_entrypoint_wrapper.py b/airbyte-cdk/python/unit_tests/test/test_entrypoint_wrapper.py index 9f3256600b50..08797e763d37 100644 --- a/airbyte-cdk/python/unit_tests/test/test_entrypoint_wrapper.py +++ b/airbyte-cdk/python/unit_tests/test/test_entrypoint_wrapper.py @@ -9,6 +9,7 @@ from airbyte_cdk.sources.abstract_source import AbstractSource from airbyte_cdk.test.entrypoint_wrapper import read +from airbyte_cdk.test.state_builder import StateBuilder from airbyte_protocol.models import ( AirbyteAnalyticsTraceMessage, AirbyteErrorTraceMessage, @@ -91,7 +92,7 @@ def _a_status_message(stream_name: str, status: AirbyteStreamStatus) -> AirbyteM ] } ) -_A_STATE = {"state_key": "state_value"} +_A_STATE = StateBuilder().with_stream_state(_A_STREAM_NAME, {"state_key": "state_value"}).build() _A_LOG_MESSAGE = "a log message" From b7a39ceb3424933c438e58e7fde45fa86b8825ad Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 30 Jan 2024 15:28:52 +0100 Subject: [PATCH 22/96] Publish to pypi batch 2 (#34656) --- .../connectors/source-babelforce/metadata.yaml | 4 ++++ .../connectors/source-bamboo-hr/metadata.yaml | 4 ++++ .../connectors/source-bigcommerce/metadata.yaml | 4 ++++ .../connectors/source-bing-ads/metadata.yaml | 5 +++++ .../connectors/source-braintree/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-braze/metadata.yaml | 4 ++++ .../connectors/source-breezometer/metadata.yaml | 4 ++++ .../connectors/source-callrail/metadata.yaml | 4 ++++ .../connectors/source-captain-data/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-cart/metadata.yaml | 4 ++++ .../connectors/source-chargebee/metadata.yaml | 4 ++++ .../connectors/source-chargify/metadata.yaml | 4 ++++ .../connectors/source-chartmogul/metadata.yaml | 4 ++++ .../connectors/source-clickup-api/metadata.yaml | 4 ++++ .../connectors/source-clockify/metadata.yaml | 4 ++++ .../connectors/source-close-com/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-coda/metadata.yaml | 4 ++++ .../connectors/source-coin-api/metadata.yaml | 4 ++++ .../connectors/source-coingecko-coins/metadata.yaml | 4 ++++ .../connectors/source-coinmarketcap/metadata.yaml | 4 ++++ 20 files changed, 81 insertions(+) diff --git a/airbyte-integrations/connectors/source-babelforce/metadata.yaml b/airbyte-integrations/connectors/source-babelforce/metadata.yaml index f4f4e35ea4ba..5e2159e65e2b 100644 --- a/airbyte-integrations/connectors/source-babelforce/metadata.yaml +++ b/airbyte-integrations/connectors/source-babelforce/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - ${region}.babelforce.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-babelforce registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-bamboo-hr/metadata.yaml b/airbyte-integrations/connectors/source-bamboo-hr/metadata.yaml index ea2b009e281f..f5ac3969826f 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/metadata.yaml +++ b/airbyte-integrations/connectors/source-bamboo-hr/metadata.yaml @@ -12,6 +12,10 @@ data: icon: bamboohr.svg license: MIT name: BambooHR + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-bamboo-hr registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-bigcommerce/metadata.yaml b/airbyte-integrations/connectors/source-bigcommerce/metadata.yaml index fd54e9032d49..9a2a63d4a3e0 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/metadata.yaml +++ b/airbyte-integrations/connectors/source-bigcommerce/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.bigcommerce.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-bigcommerce registries: oss: enabled: false diff --git a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml index 5420384149a1..644186bafbf2 100644 --- a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml @@ -23,6 +23,11 @@ data: icon: bingads.svg license: MIT name: Bing Ads + remoteRegistries: + pypi: + enabled: false + # TODO: Set enabled=true after `airbyte-lib-validate-source` is passing. + packageName: airbyte-source-bing-ads registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-braintree/metadata.yaml b/airbyte-integrations/connectors/source-braintree/metadata.yaml index d2f0ca52f2bb..5d948eee41b6 100644 --- a/airbyte-integrations/connectors/source-braintree/metadata.yaml +++ b/airbyte-integrations/connectors/source-braintree/metadata.yaml @@ -12,6 +12,10 @@ data: icon: braintree.svg license: MIT name: Braintree + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-braintree registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-braze/metadata.yaml b/airbyte-integrations/connectors/source-braze/metadata.yaml index bde782b0dc7f..e1fdcf539fd3 100644 --- a/airbyte-integrations/connectors/source-braze/metadata.yaml +++ b/airbyte-integrations/connectors/source-braze/metadata.yaml @@ -12,6 +12,10 @@ data: icon: braze.svg license: MIT name: Braze + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-braze registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-breezometer/metadata.yaml b/airbyte-integrations/connectors/source-breezometer/metadata.yaml index b325179a45ca..16796af3c94f 100644 --- a/airbyte-integrations/connectors/source-breezometer/metadata.yaml +++ b/airbyte-integrations/connectors/source-breezometer/metadata.yaml @@ -8,6 +8,10 @@ data: icon: breezometer.svg license: MIT name: Breezometer + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-breezometer registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-callrail/metadata.yaml b/airbyte-integrations/connectors/source-callrail/metadata.yaml index d876058aa9c0..f0e46fadc35a 100644 --- a/airbyte-integrations/connectors/source-callrail/metadata.yaml +++ b/airbyte-integrations/connectors/source-callrail/metadata.yaml @@ -8,6 +8,10 @@ data: icon: callrail.svg license: MIT name: CallRail + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-callrail registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-captain-data/metadata.yaml b/airbyte-integrations/connectors/source-captain-data/metadata.yaml index 715cb3f3c9c1..e5f9108699ad 100644 --- a/airbyte-integrations/connectors/source-captain-data/metadata.yaml +++ b/airbyte-integrations/connectors/source-captain-data/metadata.yaml @@ -8,6 +8,10 @@ data: icon: captain-data.svg license: MIT name: Captain Data + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-captain-data registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-cart/metadata.yaml b/airbyte-integrations/connectors/source-cart/metadata.yaml index d73ebd080df9..113417c673b0 100644 --- a/airbyte-integrations/connectors/source-cart/metadata.yaml +++ b/airbyte-integrations/connectors/source-cart/metadata.yaml @@ -8,6 +8,10 @@ data: icon: cart.svg license: MIT name: Cart.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-cart registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-chargebee/metadata.yaml b/airbyte-integrations/connectors/source-chargebee/metadata.yaml index 9fa19f17b91f..588d52fc5194 100644 --- a/airbyte-integrations/connectors/source-chargebee/metadata.yaml +++ b/airbyte-integrations/connectors/source-chargebee/metadata.yaml @@ -17,6 +17,10 @@ data: icon: chargebee.svg license: MIT name: Chargebee + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-chargebee registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-chargify/metadata.yaml b/airbyte-integrations/connectors/source-chargify/metadata.yaml index cc211d82e70c..ea6cf3005986 100644 --- a/airbyte-integrations/connectors/source-chargify/metadata.yaml +++ b/airbyte-integrations/connectors/source-chargify/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - ${domain} + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-chargify registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-chartmogul/metadata.yaml b/airbyte-integrations/connectors/source-chartmogul/metadata.yaml index 42e6a35f9b9f..3a68eecb886c 100644 --- a/airbyte-integrations/connectors/source-chartmogul/metadata.yaml +++ b/airbyte-integrations/connectors/source-chartmogul/metadata.yaml @@ -20,6 +20,10 @@ data: icon: chartmogul.svg license: MIT name: Chartmogul + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-chartmogul registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-clickup-api/metadata.yaml b/airbyte-integrations/connectors/source-clickup-api/metadata.yaml index a59f15841d5d..dc76068faa84 100644 --- a/airbyte-integrations/connectors/source-clickup-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-clickup-api/metadata.yaml @@ -8,6 +8,10 @@ data: icon: clickup.svg license: MIT name: ClickUp + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-clickup-api registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-clockify/metadata.yaml b/airbyte-integrations/connectors/source-clockify/metadata.yaml index 90baae2689d8..be8435177779 100644 --- a/airbyte-integrations/connectors/source-clockify/metadata.yaml +++ b/airbyte-integrations/connectors/source-clockify/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.clockify.me + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-clockify registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-close-com/metadata.yaml b/airbyte-integrations/connectors/source-close-com/metadata.yaml index ccb09a7823a9..ad645a347058 100644 --- a/airbyte-integrations/connectors/source-close-com/metadata.yaml +++ b/airbyte-integrations/connectors/source-close-com/metadata.yaml @@ -15,6 +15,10 @@ data: icon: close.svg license: MIT name: Close.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-close-com registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-coda/metadata.yaml b/airbyte-integrations/connectors/source-coda/metadata.yaml index c16c1748fef9..4fcb66aba68c 100644 --- a/airbyte-integrations/connectors/source-coda/metadata.yaml +++ b/airbyte-integrations/connectors/source-coda/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - https://coda.io/ + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-coda registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-coin-api/metadata.yaml b/airbyte-integrations/connectors/source-coin-api/metadata.yaml index fc65c06c61c2..039073ce42f8 100644 --- a/airbyte-integrations/connectors/source-coin-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-coin-api/metadata.yaml @@ -8,6 +8,10 @@ data: icon: coinapi.svg license: MIT name: Coin API + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-coin-api registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-coingecko-coins/metadata.yaml b/airbyte-integrations/connectors/source-coingecko-coins/metadata.yaml index b7619d147910..ac0035a6f5ff 100644 --- a/airbyte-integrations/connectors/source-coingecko-coins/metadata.yaml +++ b/airbyte-integrations/connectors/source-coingecko-coins/metadata.yaml @@ -8,6 +8,10 @@ data: icon: coingeckocoins.svg license: MIT name: CoinGecko Coins + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-coingecko-coins registries: cloud: enabled: false # Did not pass acceptance tests diff --git a/airbyte-integrations/connectors/source-coinmarketcap/metadata.yaml b/airbyte-integrations/connectors/source-coinmarketcap/metadata.yaml index a73b7184a363..d12360ad7599 100644 --- a/airbyte-integrations/connectors/source-coinmarketcap/metadata.yaml +++ b/airbyte-integrations/connectors/source-coinmarketcap/metadata.yaml @@ -8,6 +8,10 @@ data: icon: coinmarketcap.svg license: MIT name: CoinMarketCap + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-coinmarketcap registries: cloud: enabled: true From 6a6b89cd814ea9cf21ff98f0c08b18f671b59406 Mon Sep 17 00:00:00 2001 From: Maxime Carbonneau-Leclerc <3360483+maxi297@users.noreply.github.com> Date: Tue, 30 Jan 2024 10:03:00 -0500 Subject: [PATCH 23/96] Fix Cursor interface change + FileBased change at the same time (#34653) --- .../sources/file_based/stream/concurrent/adapters.py | 1 + .../airbyte_cdk/sources/file_based/stream/concurrent/cursor.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/adapters.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/adapters.py index 731b04621705..404b7b4ab09c 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/adapters.py @@ -78,6 +78,7 @@ def create_from_stream( cursor_field=cursor_field, logger=logger, namespace=stream.namespace, + cursor=cursor, ), stream, cursor, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/cursor.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/cursor.py index a0fd47044f3d..f91e3a16f3b9 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/cursor.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/concurrent/cursor.py @@ -83,5 +83,8 @@ def observe(self, record: Record) -> None: def close_partition(self, partition: Partition) -> None: return None + def ensure_at_least_one_state_emitted(self) -> None: + return None + def set_pending_partitions(self, partitions: Iterable[Partition]) -> None: return None From e3c14c78e1bfecc02f2edb0192e599543b932d47 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 30 Jan 2024 16:06:23 +0100 Subject: [PATCH 24/96] Publish to pypi batch3 (#34657) --- airbyte-integrations/connectors/source-commcare/metadata.yaml | 4 ++++ .../connectors/source-commercetools/metadata.yaml | 4 ++++ .../connectors/source-configcat/metadata.yaml | 4 ++++ .../connectors/source-confluence/metadata.yaml | 4 ++++ .../connectors/source-convertkit/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-convex/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-copper/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-courier/metadata.yaml | 4 ++++ .../connectors/source-customer-io/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-datadog/metadata.yaml | 4 ++++ .../connectors/source-datascope/metadata.yaml | 4 ++++ .../connectors/source-delighted/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-dixa/metadata.yaml | 4 ++++ .../connectors/source-dockerhub/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-dremio/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-drift/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-dv-360/metadata.yaml | 4 ++++ .../connectors/source-emailoctopus/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-everhour/metadata.yaml | 4 ++++ .../connectors/source-exchange-rates/metadata.yaml | 4 ++++ 20 files changed, 80 insertions(+) diff --git a/airbyte-integrations/connectors/source-commcare/metadata.yaml b/airbyte-integrations/connectors/source-commcare/metadata.yaml index e1bc67ca7ec8..3e78f837b0e1 100644 --- a/airbyte-integrations/connectors/source-commcare/metadata.yaml +++ b/airbyte-integrations/connectors/source-commcare/metadata.yaml @@ -7,6 +7,10 @@ data: githubIssueLabel: source-commcare license: MIT name: Commcare + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-commcare registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-commercetools/metadata.yaml b/airbyte-integrations/connectors/source-commercetools/metadata.yaml index 5af0802c2c0e..f28b88a916c9 100644 --- a/airbyte-integrations/connectors/source-commercetools/metadata.yaml +++ b/airbyte-integrations/connectors/source-commercetools/metadata.yaml @@ -3,6 +3,10 @@ data: hosts: - auth.${region}.${host}.commercetools.com - api.${region}.${host}.commercetools.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-commercetools registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-configcat/metadata.yaml b/airbyte-integrations/connectors/source-configcat/metadata.yaml index b4725144c2f1..3c1f707095be 100644 --- a/airbyte-integrations/connectors/source-configcat/metadata.yaml +++ b/airbyte-integrations/connectors/source-configcat/metadata.yaml @@ -8,6 +8,10 @@ data: icon: configcat.svg license: MIT name: ConfigCat + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-configcat registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-confluence/metadata.yaml b/airbyte-integrations/connectors/source-confluence/metadata.yaml index 87d324b5ebec..be0b48fb334a 100644 --- a/airbyte-integrations/connectors/source-confluence/metadata.yaml +++ b/airbyte-integrations/connectors/source-confluence/metadata.yaml @@ -5,6 +5,10 @@ data: allowedHosts: hosts: - ${subdomain}.atlassian.net + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-confluence registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-convertkit/metadata.yaml b/airbyte-integrations/connectors/source-convertkit/metadata.yaml index 66d558a615f6..8a0327f0580c 100644 --- a/airbyte-integrations/connectors/source-convertkit/metadata.yaml +++ b/airbyte-integrations/connectors/source-convertkit/metadata.yaml @@ -8,6 +8,10 @@ data: icon: convertkit.svg license: MIT name: ConvertKit + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-convertkit registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-convex/metadata.yaml b/airbyte-integrations/connectors/source-convex/metadata.yaml index b58c7a7b40dd..14d34525a641 100644 --- a/airbyte-integrations/connectors/source-convex/metadata.yaml +++ b/airbyte-integrations/connectors/source-convex/metadata.yaml @@ -8,6 +8,10 @@ data: icon: convex.svg license: MIT name: Convex + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-convex registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-copper/metadata.yaml b/airbyte-integrations/connectors/source-copper/metadata.yaml index 283dd98e3a15..6298afd8c8b5 100644 --- a/airbyte-integrations/connectors/source-copper/metadata.yaml +++ b/airbyte-integrations/connectors/source-copper/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - https://api.copper.com/ + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-copper registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-courier/metadata.yaml b/airbyte-integrations/connectors/source-courier/metadata.yaml index d2badc1d80db..fac285152d97 100644 --- a/airbyte-integrations/connectors/source-courier/metadata.yaml +++ b/airbyte-integrations/connectors/source-courier/metadata.yaml @@ -8,6 +8,10 @@ data: icon: courier.svg license: MIT name: Courier + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-courier registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-customer-io/metadata.yaml b/airbyte-integrations/connectors/source-customer-io/metadata.yaml index 3cf0252a2512..6bb0d1e26176 100644 --- a/airbyte-integrations/connectors/source-customer-io/metadata.yaml +++ b/airbyte-integrations/connectors/source-customer-io/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - https://api.customer.io/v1/ + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-customer-io registries: oss: enabled: false diff --git a/airbyte-integrations/connectors/source-datadog/metadata.yaml b/airbyte-integrations/connectors/source-datadog/metadata.yaml index 88bef85da8ba..da27ba470328 100644 --- a/airbyte-integrations/connectors/source-datadog/metadata.yaml +++ b/airbyte-integrations/connectors/source-datadog/metadata.yaml @@ -6,6 +6,10 @@ data: - us5.datadoghq.com - datadoghq.eu - ddog-gov.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-datadog registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-datascope/metadata.yaml b/airbyte-integrations/connectors/source-datascope/metadata.yaml index 2d61ac39b220..2649a980d2dd 100644 --- a/airbyte-integrations/connectors/source-datascope/metadata.yaml +++ b/airbyte-integrations/connectors/source-datascope/metadata.yaml @@ -8,6 +8,10 @@ data: icon: datascope.svg license: MIT name: Datascope + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-datascope registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-delighted/metadata.yaml b/airbyte-integrations/connectors/source-delighted/metadata.yaml index 2a050c2b4814..c9d006b900ee 100644 --- a/airbyte-integrations/connectors/source-delighted/metadata.yaml +++ b/airbyte-integrations/connectors/source-delighted/metadata.yaml @@ -15,6 +15,10 @@ data: icon: delighted.svg license: MIT name: Delighted + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-delighted registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-dixa/metadata.yaml b/airbyte-integrations/connectors/source-dixa/metadata.yaml index 9d45d0bb45f7..bb4821ee8c81 100644 --- a/airbyte-integrations/connectors/source-dixa/metadata.yaml +++ b/airbyte-integrations/connectors/source-dixa/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - exports.dixa.io + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-dixa registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-dockerhub/metadata.yaml b/airbyte-integrations/connectors/source-dockerhub/metadata.yaml index e0b403736301..7800c7d8f8d9 100644 --- a/airbyte-integrations/connectors/source-dockerhub/metadata.yaml +++ b/airbyte-integrations/connectors/source-dockerhub/metadata.yaml @@ -3,6 +3,10 @@ data: hosts: - hub.docker.com - auth.docker.io + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-dockerhub registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-dremio/metadata.yaml b/airbyte-integrations/connectors/source-dremio/metadata.yaml index 4315e72f6be3..406e3707fdda 100644 --- a/airbyte-integrations/connectors/source-dremio/metadata.yaml +++ b/airbyte-integrations/connectors/source-dremio/metadata.yaml @@ -8,6 +8,10 @@ data: icon: dremio.svg license: MIT name: Dremio + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-dremio registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-drift/metadata.yaml b/airbyte-integrations/connectors/source-drift/metadata.yaml index 12e3531ac2c3..e22ab893eae5 100644 --- a/airbyte-integrations/connectors/source-drift/metadata.yaml +++ b/airbyte-integrations/connectors/source-drift/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - https://driftapi.com/ + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-drift registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-dv-360/metadata.yaml b/airbyte-integrations/connectors/source-dv-360/metadata.yaml index acaa90cec05d..8092f2fce14c 100644 --- a/airbyte-integrations/connectors/source-dv-360/metadata.yaml +++ b/airbyte-integrations/connectors/source-dv-360/metadata.yaml @@ -8,6 +8,10 @@ data: icon: dv360.svg license: MIT name: DV 360 + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-dv-360 registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-emailoctopus/metadata.yaml b/airbyte-integrations/connectors/source-emailoctopus/metadata.yaml index 6b3d33420107..55a5037b7c05 100644 --- a/airbyte-integrations/connectors/source-emailoctopus/metadata.yaml +++ b/airbyte-integrations/connectors/source-emailoctopus/metadata.yaml @@ -8,6 +8,10 @@ data: icon: emailoctopus.svg license: MIT name: EmailOctopus + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-emailoctopus registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-everhour/metadata.yaml b/airbyte-integrations/connectors/source-everhour/metadata.yaml index a41a9283a3d1..edc9128f054f 100644 --- a/airbyte-integrations/connectors/source-everhour/metadata.yaml +++ b/airbyte-integrations/connectors/source-everhour/metadata.yaml @@ -11,6 +11,10 @@ data: icon: everhour.svg license: MIT name: Everhour + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-everhour registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml b/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml index 20d3fa38cb7c..32efbf5e53ff 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml +++ b/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml @@ -3,6 +3,10 @@ data: hosts: - ${subdomain}.apilayer.com - apilayer.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-exchange-rates registries: oss: enabled: true From e808c5115935cd08cf199ce8a7bc8002dd3cf187 Mon Sep 17 00:00:00 2001 From: maxi297 Date: Tue, 30 Jan 2024 15:11:50 +0000 Subject: [PATCH 25/96] =?UTF-8?q?=F0=9F=A4=96=20Bump=20patch=20version=20o?= =?UTF-8?q?f=20Python=20CDK?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-cdk/python/.bumpversion.cfg | 2 +- airbyte-cdk/python/CHANGELOG.md | 3 +++ airbyte-cdk/python/Dockerfile | 2 +- airbyte-cdk/python/setup.py | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/airbyte-cdk/python/.bumpversion.cfg b/airbyte-cdk/python/.bumpversion.cfg index 3d97cc1d483b..984be1fe90a0 100644 --- a/airbyte-cdk/python/.bumpversion.cfg +++ b/airbyte-cdk/python/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.60.0 +current_version = 0.60.1 commit = False [bumpversion:file:setup.py] diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 84029d9910e2..822a7c75e2ea 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.60.1 +Emit state when no partitions are generated for ccdk and update StateBuilder + ## 0.60.0 File-based CDK: run full refresh syncs with concurrency diff --git a/airbyte-cdk/python/Dockerfile b/airbyte-cdk/python/Dockerfile index 54d7fecf5ac1..759d20f4c61a 100644 --- a/airbyte-cdk/python/Dockerfile +++ b/airbyte-cdk/python/Dockerfile @@ -32,5 +32,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] # needs to be the same as CDK -LABEL io.airbyte.version=0.60.0 +LABEL io.airbyte.version=0.60.1 LABEL io.airbyte.name=airbyte/source-declarative-manifest diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 4cae309ddfe1..9082b99b5a9b 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -36,7 +36,7 @@ name="airbyte-cdk", # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be # updated if our semver format changes such as using release candidate versions. - version="0.60.0", + version="0.60.1", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", From 15efffe16af6131c77ddde875e050328684df806 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 30 Jan 2024 16:28:42 +0100 Subject: [PATCH 26/96] Revert "Publish to pypi batch3 (#34657)" (#34659) --- airbyte-integrations/connectors/source-commcare/metadata.yaml | 4 ---- .../connectors/source-commercetools/metadata.yaml | 4 ---- .../connectors/source-configcat/metadata.yaml | 4 ---- .../connectors/source-confluence/metadata.yaml | 4 ---- .../connectors/source-convertkit/metadata.yaml | 4 ---- airbyte-integrations/connectors/source-convex/metadata.yaml | 4 ---- airbyte-integrations/connectors/source-copper/metadata.yaml | 4 ---- airbyte-integrations/connectors/source-courier/metadata.yaml | 4 ---- .../connectors/source-customer-io/metadata.yaml | 4 ---- airbyte-integrations/connectors/source-datadog/metadata.yaml | 4 ---- .../connectors/source-datascope/metadata.yaml | 4 ---- .../connectors/source-delighted/metadata.yaml | 4 ---- airbyte-integrations/connectors/source-dixa/metadata.yaml | 4 ---- .../connectors/source-dockerhub/metadata.yaml | 4 ---- airbyte-integrations/connectors/source-dremio/metadata.yaml | 4 ---- airbyte-integrations/connectors/source-drift/metadata.yaml | 4 ---- airbyte-integrations/connectors/source-dv-360/metadata.yaml | 4 ---- .../connectors/source-emailoctopus/metadata.yaml | 4 ---- airbyte-integrations/connectors/source-everhour/metadata.yaml | 4 ---- .../connectors/source-exchange-rates/metadata.yaml | 4 ---- 20 files changed, 80 deletions(-) diff --git a/airbyte-integrations/connectors/source-commcare/metadata.yaml b/airbyte-integrations/connectors/source-commcare/metadata.yaml index 3e78f837b0e1..e1bc67ca7ec8 100644 --- a/airbyte-integrations/connectors/source-commcare/metadata.yaml +++ b/airbyte-integrations/connectors/source-commcare/metadata.yaml @@ -7,10 +7,6 @@ data: githubIssueLabel: source-commcare license: MIT name: Commcare - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-commcare registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-commercetools/metadata.yaml b/airbyte-integrations/connectors/source-commercetools/metadata.yaml index f28b88a916c9..5af0802c2c0e 100644 --- a/airbyte-integrations/connectors/source-commercetools/metadata.yaml +++ b/airbyte-integrations/connectors/source-commercetools/metadata.yaml @@ -3,10 +3,6 @@ data: hosts: - auth.${region}.${host}.commercetools.com - api.${region}.${host}.commercetools.com - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-commercetools registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-configcat/metadata.yaml b/airbyte-integrations/connectors/source-configcat/metadata.yaml index 3c1f707095be..b4725144c2f1 100644 --- a/airbyte-integrations/connectors/source-configcat/metadata.yaml +++ b/airbyte-integrations/connectors/source-configcat/metadata.yaml @@ -8,10 +8,6 @@ data: icon: configcat.svg license: MIT name: ConfigCat - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-configcat registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-confluence/metadata.yaml b/airbyte-integrations/connectors/source-confluence/metadata.yaml index be0b48fb334a..87d324b5ebec 100644 --- a/airbyte-integrations/connectors/source-confluence/metadata.yaml +++ b/airbyte-integrations/connectors/source-confluence/metadata.yaml @@ -5,10 +5,6 @@ data: allowedHosts: hosts: - ${subdomain}.atlassian.net - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-confluence registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-convertkit/metadata.yaml b/airbyte-integrations/connectors/source-convertkit/metadata.yaml index 8a0327f0580c..66d558a615f6 100644 --- a/airbyte-integrations/connectors/source-convertkit/metadata.yaml +++ b/airbyte-integrations/connectors/source-convertkit/metadata.yaml @@ -8,10 +8,6 @@ data: icon: convertkit.svg license: MIT name: ConvertKit - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-convertkit registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-convex/metadata.yaml b/airbyte-integrations/connectors/source-convex/metadata.yaml index 14d34525a641..b58c7a7b40dd 100644 --- a/airbyte-integrations/connectors/source-convex/metadata.yaml +++ b/airbyte-integrations/connectors/source-convex/metadata.yaml @@ -8,10 +8,6 @@ data: icon: convex.svg license: MIT name: Convex - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-convex registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-copper/metadata.yaml b/airbyte-integrations/connectors/source-copper/metadata.yaml index 6298afd8c8b5..283dd98e3a15 100644 --- a/airbyte-integrations/connectors/source-copper/metadata.yaml +++ b/airbyte-integrations/connectors/source-copper/metadata.yaml @@ -2,10 +2,6 @@ data: allowedHosts: hosts: - https://api.copper.com/ - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-copper registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-courier/metadata.yaml b/airbyte-integrations/connectors/source-courier/metadata.yaml index fac285152d97..d2badc1d80db 100644 --- a/airbyte-integrations/connectors/source-courier/metadata.yaml +++ b/airbyte-integrations/connectors/source-courier/metadata.yaml @@ -8,10 +8,6 @@ data: icon: courier.svg license: MIT name: Courier - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-courier registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-customer-io/metadata.yaml b/airbyte-integrations/connectors/source-customer-io/metadata.yaml index 6bb0d1e26176..3cf0252a2512 100644 --- a/airbyte-integrations/connectors/source-customer-io/metadata.yaml +++ b/airbyte-integrations/connectors/source-customer-io/metadata.yaml @@ -2,10 +2,6 @@ data: allowedHosts: hosts: - https://api.customer.io/v1/ - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-customer-io registries: oss: enabled: false diff --git a/airbyte-integrations/connectors/source-datadog/metadata.yaml b/airbyte-integrations/connectors/source-datadog/metadata.yaml index da27ba470328..88bef85da8ba 100644 --- a/airbyte-integrations/connectors/source-datadog/metadata.yaml +++ b/airbyte-integrations/connectors/source-datadog/metadata.yaml @@ -6,10 +6,6 @@ data: - us5.datadoghq.com - datadoghq.eu - ddog-gov.com - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-datadog registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-datascope/metadata.yaml b/airbyte-integrations/connectors/source-datascope/metadata.yaml index 2649a980d2dd..2d61ac39b220 100644 --- a/airbyte-integrations/connectors/source-datascope/metadata.yaml +++ b/airbyte-integrations/connectors/source-datascope/metadata.yaml @@ -8,10 +8,6 @@ data: icon: datascope.svg license: MIT name: Datascope - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-datascope registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-delighted/metadata.yaml b/airbyte-integrations/connectors/source-delighted/metadata.yaml index c9d006b900ee..2a050c2b4814 100644 --- a/airbyte-integrations/connectors/source-delighted/metadata.yaml +++ b/airbyte-integrations/connectors/source-delighted/metadata.yaml @@ -15,10 +15,6 @@ data: icon: delighted.svg license: MIT name: Delighted - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-delighted registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-dixa/metadata.yaml b/airbyte-integrations/connectors/source-dixa/metadata.yaml index bb4821ee8c81..9d45d0bb45f7 100644 --- a/airbyte-integrations/connectors/source-dixa/metadata.yaml +++ b/airbyte-integrations/connectors/source-dixa/metadata.yaml @@ -2,10 +2,6 @@ data: allowedHosts: hosts: - exports.dixa.io - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-dixa registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-dockerhub/metadata.yaml b/airbyte-integrations/connectors/source-dockerhub/metadata.yaml index 7800c7d8f8d9..e0b403736301 100644 --- a/airbyte-integrations/connectors/source-dockerhub/metadata.yaml +++ b/airbyte-integrations/connectors/source-dockerhub/metadata.yaml @@ -3,10 +3,6 @@ data: hosts: - hub.docker.com - auth.docker.io - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-dockerhub registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-dremio/metadata.yaml b/airbyte-integrations/connectors/source-dremio/metadata.yaml index 406e3707fdda..4315e72f6be3 100644 --- a/airbyte-integrations/connectors/source-dremio/metadata.yaml +++ b/airbyte-integrations/connectors/source-dremio/metadata.yaml @@ -8,10 +8,6 @@ data: icon: dremio.svg license: MIT name: Dremio - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-dremio registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-drift/metadata.yaml b/airbyte-integrations/connectors/source-drift/metadata.yaml index e22ab893eae5..12e3531ac2c3 100644 --- a/airbyte-integrations/connectors/source-drift/metadata.yaml +++ b/airbyte-integrations/connectors/source-drift/metadata.yaml @@ -2,10 +2,6 @@ data: allowedHosts: hosts: - https://driftapi.com/ - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-drift registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-dv-360/metadata.yaml b/airbyte-integrations/connectors/source-dv-360/metadata.yaml index 8092f2fce14c..acaa90cec05d 100644 --- a/airbyte-integrations/connectors/source-dv-360/metadata.yaml +++ b/airbyte-integrations/connectors/source-dv-360/metadata.yaml @@ -8,10 +8,6 @@ data: icon: dv360.svg license: MIT name: DV 360 - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-dv-360 registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-emailoctopus/metadata.yaml b/airbyte-integrations/connectors/source-emailoctopus/metadata.yaml index 55a5037b7c05..6b3d33420107 100644 --- a/airbyte-integrations/connectors/source-emailoctopus/metadata.yaml +++ b/airbyte-integrations/connectors/source-emailoctopus/metadata.yaml @@ -8,10 +8,6 @@ data: icon: emailoctopus.svg license: MIT name: EmailOctopus - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-emailoctopus registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-everhour/metadata.yaml b/airbyte-integrations/connectors/source-everhour/metadata.yaml index edc9128f054f..a41a9283a3d1 100644 --- a/airbyte-integrations/connectors/source-everhour/metadata.yaml +++ b/airbyte-integrations/connectors/source-everhour/metadata.yaml @@ -11,10 +11,6 @@ data: icon: everhour.svg license: MIT name: Everhour - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-everhour registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml b/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml index 32efbf5e53ff..20d3fa38cb7c 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml +++ b/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml @@ -3,10 +3,6 @@ data: hosts: - ${subdomain}.apilayer.com - apilayer.com - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-exchange-rates registries: oss: enabled: true From ae3a028b435ea0b8590dd522491c5a5e0a38b048 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 30 Jan 2024 18:02:55 +0100 Subject: [PATCH 27/96] airbyte-lib: Prepare for published connectors (#34651) --- airbyte-lib/airbyte_lib/_executor.py | 6 +++--- .../tests/integration_tests/fixtures/source-broken/setup.py | 4 ++-- .../tests/integration_tests/fixtures/source-test/setup.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/airbyte-lib/airbyte_lib/_executor.py b/airbyte-lib/airbyte_lib/_executor.py index a43d56249163..45cb03cf29df 100644 --- a/airbyte-lib/airbyte_lib/_executor.py +++ b/airbyte-lib/airbyte_lib/_executor.py @@ -147,8 +147,7 @@ def __init__( # This is a temporary install path that will be replaced with a proper package # name once they are published. - # TODO: Replace with `f"airbyte-{self.name}"` - self.pip_url = pip_url or f"../airbyte-integrations/connectors/{self.name}" + self.pip_url = pip_url or f"airbyte-{self.name}" self.install_root = install_root or Path.cwd() def _get_venv_name(self) -> str: @@ -239,11 +238,12 @@ def _get_installed_version( return None try: + package_name = f"airbyte-{connector_name}" return subprocess.check_output( [ self.interpreter_path, "-c", - f"from importlib.metadata import version; print(version('{connector_name}'))", + f"from importlib.metadata import version; print(version('{package_name}'))", ], universal_newlines=True, ).strip() diff --git a/airbyte-lib/tests/integration_tests/fixtures/source-broken/setup.py b/airbyte-lib/tests/integration_tests/fixtures/source-broken/setup.py index 0a565f98e08b..516112718b7e 100644 --- a/airbyte-lib/tests/integration_tests/fixtures/source-broken/setup.py +++ b/airbyte-lib/tests/integration_tests/fixtures/source-broken/setup.py @@ -3,10 +3,10 @@ # -from setuptools import find_packages, setup +from setuptools import setup setup( - name="source_broken", + name="airbyte-source-broken", version="0.0.1", description="Test Soutce", author="Airbyte", diff --git a/airbyte-lib/tests/integration_tests/fixtures/source-test/setup.py b/airbyte-lib/tests/integration_tests/fixtures/source-test/setup.py index 348e0c28a254..0035f1eda76a 100644 --- a/airbyte-lib/tests/integration_tests/fixtures/source-test/setup.py +++ b/airbyte-lib/tests/integration_tests/fixtures/source-test/setup.py @@ -3,10 +3,10 @@ # -from setuptools import find_packages, setup +from setuptools import setup setup( - name="source_test", + name="airbyte-source-test", version="0.0.1", description="Test Soutce", author="Airbyte", From 4ccdf362d59f85b5946903915bc3931f6e3a6a4b Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 30 Jan 2024 18:30:48 +0100 Subject: [PATCH 28/96] Recreate pypi publish batch3 (#34660) --- airbyte-integrations/connectors/source-commcare/metadata.yaml | 4 ++++ .../connectors/source-commercetools/metadata.yaml | 4 ++++ .../connectors/source-configcat/metadata.yaml | 4 ++++ .../connectors/source-confluence/metadata.yaml | 4 ++++ .../connectors/source-convertkit/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-convex/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-copper/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-courier/metadata.yaml | 4 ++++ .../connectors/source-customer-io/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-datadog/metadata.yaml | 4 ++++ .../connectors/source-datascope/metadata.yaml | 4 ++++ .../connectors/source-delighted/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-dixa/metadata.yaml | 4 ++++ .../connectors/source-dockerhub/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-dremio/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-drift/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-dv-360/metadata.yaml | 4 ++++ .../connectors/source-emailoctopus/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-everhour/metadata.yaml | 4 ++++ .../connectors/source-exchange-rates/metadata.yaml | 4 ++++ 20 files changed, 80 insertions(+) diff --git a/airbyte-integrations/connectors/source-commcare/metadata.yaml b/airbyte-integrations/connectors/source-commcare/metadata.yaml index e1bc67ca7ec8..3e78f837b0e1 100644 --- a/airbyte-integrations/connectors/source-commcare/metadata.yaml +++ b/airbyte-integrations/connectors/source-commcare/metadata.yaml @@ -7,6 +7,10 @@ data: githubIssueLabel: source-commcare license: MIT name: Commcare + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-commcare registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-commercetools/metadata.yaml b/airbyte-integrations/connectors/source-commercetools/metadata.yaml index 5af0802c2c0e..f28b88a916c9 100644 --- a/airbyte-integrations/connectors/source-commercetools/metadata.yaml +++ b/airbyte-integrations/connectors/source-commercetools/metadata.yaml @@ -3,6 +3,10 @@ data: hosts: - auth.${region}.${host}.commercetools.com - api.${region}.${host}.commercetools.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-commercetools registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-configcat/metadata.yaml b/airbyte-integrations/connectors/source-configcat/metadata.yaml index b4725144c2f1..3c1f707095be 100644 --- a/airbyte-integrations/connectors/source-configcat/metadata.yaml +++ b/airbyte-integrations/connectors/source-configcat/metadata.yaml @@ -8,6 +8,10 @@ data: icon: configcat.svg license: MIT name: ConfigCat + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-configcat registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-confluence/metadata.yaml b/airbyte-integrations/connectors/source-confluence/metadata.yaml index 87d324b5ebec..be0b48fb334a 100644 --- a/airbyte-integrations/connectors/source-confluence/metadata.yaml +++ b/airbyte-integrations/connectors/source-confluence/metadata.yaml @@ -5,6 +5,10 @@ data: allowedHosts: hosts: - ${subdomain}.atlassian.net + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-confluence registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-convertkit/metadata.yaml b/airbyte-integrations/connectors/source-convertkit/metadata.yaml index 66d558a615f6..8a0327f0580c 100644 --- a/airbyte-integrations/connectors/source-convertkit/metadata.yaml +++ b/airbyte-integrations/connectors/source-convertkit/metadata.yaml @@ -8,6 +8,10 @@ data: icon: convertkit.svg license: MIT name: ConvertKit + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-convertkit registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-convex/metadata.yaml b/airbyte-integrations/connectors/source-convex/metadata.yaml index b58c7a7b40dd..14d34525a641 100644 --- a/airbyte-integrations/connectors/source-convex/metadata.yaml +++ b/airbyte-integrations/connectors/source-convex/metadata.yaml @@ -8,6 +8,10 @@ data: icon: convex.svg license: MIT name: Convex + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-convex registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-copper/metadata.yaml b/airbyte-integrations/connectors/source-copper/metadata.yaml index 283dd98e3a15..6298afd8c8b5 100644 --- a/airbyte-integrations/connectors/source-copper/metadata.yaml +++ b/airbyte-integrations/connectors/source-copper/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - https://api.copper.com/ + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-copper registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-courier/metadata.yaml b/airbyte-integrations/connectors/source-courier/metadata.yaml index d2badc1d80db..fac285152d97 100644 --- a/airbyte-integrations/connectors/source-courier/metadata.yaml +++ b/airbyte-integrations/connectors/source-courier/metadata.yaml @@ -8,6 +8,10 @@ data: icon: courier.svg license: MIT name: Courier + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-courier registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-customer-io/metadata.yaml b/airbyte-integrations/connectors/source-customer-io/metadata.yaml index 3cf0252a2512..6bb0d1e26176 100644 --- a/airbyte-integrations/connectors/source-customer-io/metadata.yaml +++ b/airbyte-integrations/connectors/source-customer-io/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - https://api.customer.io/v1/ + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-customer-io registries: oss: enabled: false diff --git a/airbyte-integrations/connectors/source-datadog/metadata.yaml b/airbyte-integrations/connectors/source-datadog/metadata.yaml index 88bef85da8ba..da27ba470328 100644 --- a/airbyte-integrations/connectors/source-datadog/metadata.yaml +++ b/airbyte-integrations/connectors/source-datadog/metadata.yaml @@ -6,6 +6,10 @@ data: - us5.datadoghq.com - datadoghq.eu - ddog-gov.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-datadog registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-datascope/metadata.yaml b/airbyte-integrations/connectors/source-datascope/metadata.yaml index 2d61ac39b220..2649a980d2dd 100644 --- a/airbyte-integrations/connectors/source-datascope/metadata.yaml +++ b/airbyte-integrations/connectors/source-datascope/metadata.yaml @@ -8,6 +8,10 @@ data: icon: datascope.svg license: MIT name: Datascope + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-datascope registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-delighted/metadata.yaml b/airbyte-integrations/connectors/source-delighted/metadata.yaml index 2a050c2b4814..c9d006b900ee 100644 --- a/airbyte-integrations/connectors/source-delighted/metadata.yaml +++ b/airbyte-integrations/connectors/source-delighted/metadata.yaml @@ -15,6 +15,10 @@ data: icon: delighted.svg license: MIT name: Delighted + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-delighted registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-dixa/metadata.yaml b/airbyte-integrations/connectors/source-dixa/metadata.yaml index 9d45d0bb45f7..bb4821ee8c81 100644 --- a/airbyte-integrations/connectors/source-dixa/metadata.yaml +++ b/airbyte-integrations/connectors/source-dixa/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - exports.dixa.io + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-dixa registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-dockerhub/metadata.yaml b/airbyte-integrations/connectors/source-dockerhub/metadata.yaml index e0b403736301..7800c7d8f8d9 100644 --- a/airbyte-integrations/connectors/source-dockerhub/metadata.yaml +++ b/airbyte-integrations/connectors/source-dockerhub/metadata.yaml @@ -3,6 +3,10 @@ data: hosts: - hub.docker.com - auth.docker.io + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-dockerhub registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-dremio/metadata.yaml b/airbyte-integrations/connectors/source-dremio/metadata.yaml index 4315e72f6be3..406e3707fdda 100644 --- a/airbyte-integrations/connectors/source-dremio/metadata.yaml +++ b/airbyte-integrations/connectors/source-dremio/metadata.yaml @@ -8,6 +8,10 @@ data: icon: dremio.svg license: MIT name: Dremio + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-dremio registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-drift/metadata.yaml b/airbyte-integrations/connectors/source-drift/metadata.yaml index 12e3531ac2c3..e22ab893eae5 100644 --- a/airbyte-integrations/connectors/source-drift/metadata.yaml +++ b/airbyte-integrations/connectors/source-drift/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - https://driftapi.com/ + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-drift registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-dv-360/metadata.yaml b/airbyte-integrations/connectors/source-dv-360/metadata.yaml index acaa90cec05d..8092f2fce14c 100644 --- a/airbyte-integrations/connectors/source-dv-360/metadata.yaml +++ b/airbyte-integrations/connectors/source-dv-360/metadata.yaml @@ -8,6 +8,10 @@ data: icon: dv360.svg license: MIT name: DV 360 + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-dv-360 registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-emailoctopus/metadata.yaml b/airbyte-integrations/connectors/source-emailoctopus/metadata.yaml index 6b3d33420107..55a5037b7c05 100644 --- a/airbyte-integrations/connectors/source-emailoctopus/metadata.yaml +++ b/airbyte-integrations/connectors/source-emailoctopus/metadata.yaml @@ -8,6 +8,10 @@ data: icon: emailoctopus.svg license: MIT name: EmailOctopus + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-emailoctopus registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-everhour/metadata.yaml b/airbyte-integrations/connectors/source-everhour/metadata.yaml index a41a9283a3d1..edc9128f054f 100644 --- a/airbyte-integrations/connectors/source-everhour/metadata.yaml +++ b/airbyte-integrations/connectors/source-everhour/metadata.yaml @@ -11,6 +11,10 @@ data: icon: everhour.svg license: MIT name: Everhour + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-everhour registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml b/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml index 20d3fa38cb7c..32efbf5e53ff 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml +++ b/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml @@ -3,6 +3,10 @@ data: hosts: - ${subdomain}.apilayer.com - apilayer.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-exchange-rates registries: oss: enabled: true From 4136e9caab36926cd786615a7f68ff0406d04ab2 Mon Sep 17 00:00:00 2001 From: Ryan Waskewich <156025126+rwask@users.noreply.github.com> Date: Tue, 30 Jan 2024 12:38:00 -0500 Subject: [PATCH 29/96] Update cdc.md - Add MongoDB support (#34671) --- docs/understanding-airbyte/cdc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/understanding-airbyte/cdc.md b/docs/understanding-airbyte/cdc.md index ee42b9a460b8..fa012c6d5366 100644 --- a/docs/understanding-airbyte/cdc.md +++ b/docs/understanding-airbyte/cdc.md @@ -33,7 +33,7 @@ We add some metadata columns for CDC sources: * [Postgres](../integrations/sources/postgres.md) \(For a quick video overview of CDC on Postgres, click [here](https://www.youtube.com/watch?v=NMODvLgZvuE&ab_channel=Airbyte)\) * [MySQL](../integrations/sources/mysql.md) * [Microsoft SQL Server / MSSQL](../integrations/sources/mssql.md) - +* [MongoDB](../integrations/sources/mongodb-v2.md) \(More information on [Mongodb CDC: How to Sync in Near Real-Time](https://airbyte.com/data-engineering-resources/mongodb-cdc)\) ## Coming Soon * Oracle DB From 82b7e5d532d78703fec8de9d7a6d304e2a9e4f32 Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Tue, 30 Jan 2024 13:30:02 -0500 Subject: [PATCH 30/96] Pin file-based sources to airbyte-cdk version 0.59.2 (#34661) --- .../source-azure-blob-storage/metadata.yaml | 2 +- .../source-azure-blob-storage/setup.py | 2 +- .../connectors/source-gcs/metadata.yaml | 2 +- .../connectors/source-gcs/setup.py | 2 +- .../source-google-drive/metadata.yaml | 2 +- .../connectors/source-google-drive/setup.py | 2 +- .../source-microsoft-onedrive/metadata.yaml | 2 +- .../source-microsoft-onedrive/setup.py | 2 +- .../sources/azure-blob-storage.md | 23 ++++++++++--------- docs/integrations/sources/gcs.md | 1 + docs/integrations/sources/google-drive.md | 3 ++- .../sources/microsoft-onedrive.md | 1 + 12 files changed, 24 insertions(+), 20 deletions(-) diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml b/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml index 14923e9a8c79..eca1b2e4c108 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml +++ b/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: file connectorType: source definitionId: fdaaba68-4875-4ed9-8fcd-4ae1e0a25093 - dockerImageTag: 0.3.1 + dockerImageTag: 0.3.2 dockerRepository: airbyte/source-azure-blob-storage documentationUrl: https://docs.airbyte.com/integrations/sources/azure-blob-storage githubIssueLabel: source-azure-blob-storage diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/setup.py b/airbyte-integrations/connectors/source-azure-blob-storage/setup.py index b22b1d7c6be6..5260ac12c058 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/setup.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[file-based]>=0.57.7", + "airbyte-cdk[file-based]==0.59.2", # pinned until compatible with https://github.com/airbytehq/airbyte/pull/34411 "smart_open[azure]", "pytz", ] diff --git a/airbyte-integrations/connectors/source-gcs/metadata.yaml b/airbyte-integrations/connectors/source-gcs/metadata.yaml index 1ed2d6757783..b303f6085b7a 100644 --- a/airbyte-integrations/connectors/source-gcs/metadata.yaml +++ b/airbyte-integrations/connectors/source-gcs/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: file connectorType: source definitionId: 2a8c41ae-8c23-4be0-a73f-2ab10ca1a820 - dockerImageTag: 0.3.4 + dockerImageTag: 0.3.5 dockerRepository: airbyte/source-gcs documentationUrl: https://docs.airbyte.com/integrations/sources/gcs githubIssueLabel: source-gcs diff --git a/airbyte-integrations/connectors/source-gcs/setup.py b/airbyte-integrations/connectors/source-gcs/setup.py index 0f576ec930cc..830498574d7a 100644 --- a/airbyte-integrations/connectors/source-gcs/setup.py +++ b/airbyte-integrations/connectors/source-gcs/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[file-based]>=0.55.5", + "airbyte-cdk[file-based]==0.59.2", # pinned until compatible with https://github.com/airbytehq/airbyte/pull/34411 "google-cloud-storage==2.12.0", "smart-open[s3]==5.1.0", "pandas==1.5.3", diff --git a/airbyte-integrations/connectors/source-google-drive/metadata.yaml b/airbyte-integrations/connectors/source-google-drive/metadata.yaml index bf40629baae0..6f4839d112c9 100644 --- a/airbyte-integrations/connectors/source-google-drive/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-drive/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: file connectorType: source definitionId: 9f8dda77-1048-4368-815b-269bf54ee9b8 - dockerImageTag: 0.0.6 + dockerImageTag: 0.0.7 dockerRepository: airbyte/source-google-drive githubIssueLabel: source-google-drive icon: google-drive.svg diff --git a/airbyte-integrations/connectors/source-google-drive/setup.py b/airbyte-integrations/connectors/source-google-drive/setup.py index 0eade80fe5ca..858c7019384a 100644 --- a/airbyte-integrations/connectors/source-google-drive/setup.py +++ b/airbyte-integrations/connectors/source-google-drive/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[file-based]>=0.57.7", + "airbyte-cdk[file-based]==0.59.2", # pinned until compatible with https://github.com/airbytehq/airbyte/pull/34411 "google-api-python-client==2.104.0", "google-auth-httplib2==0.1.1", "google-auth-oauthlib==1.1.0", diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml b/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml index f1b3317b475c..b6ef12226f71 100644 --- a/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml @@ -16,7 +16,7 @@ data: connectorSubtype: api connectorType: source definitionId: 01d1c685-fd4a-4837-8f4c-93fe5a0d2188 - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 dockerRepository: airbyte/source-microsoft-onedrive githubIssueLabel: source-microsoft-onedrive icon: microsoft-onedrive.svg diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/setup.py b/airbyte-integrations/connectors/source-microsoft-onedrive/setup.py index 8709208c5be2..bc93b53a4004 100644 --- a/airbyte-integrations/connectors/source-microsoft-onedrive/setup.py +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[file-based]>=0.57.5", + "airbyte-cdk[file-based]==0.59.2", # pinned until compatible with https://github.com/airbytehq/airbyte/pull/34411 "msal~=1.25.0", "Office365-REST-Python-Client~=2.5.2", "smart-open~=6.4.0", diff --git a/docs/integrations/sources/azure-blob-storage.md b/docs/integrations/sources/azure-blob-storage.md index 0a6dc5e5e8f7..01d082c71ca0 100644 --- a/docs/integrations/sources/azure-blob-storage.md +++ b/docs/integrations/sources/azure-blob-storage.md @@ -191,14 +191,15 @@ To perform the text extraction from PDF and Docx files, the connector uses the [ ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------| -| 0.3.1 | 2024-01-10 | [34084](https://github.com/airbytehq/airbyte/pull/34084) | Fix bug for running check with document file format | -| 0.3.0 | 2023-12-14 | [33411](https://github.com/airbytehq/airbyte/pull/33411) | Bump CDK version to auto-set primary key for document file streams and support raw txt files | -| 0.2.5 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | -| 0.2.4 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Improve document file type parser | -| 0.2.3 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | -| 0.2.2 | 2023-10-30 | [31904](https://github.com/airbytehq/airbyte/pull/31904) | Update CDK to support document file types | -| 0.2.1 | 2023-10-18 | [31543](https://github.com/airbytehq/airbyte/pull/31543) | Base image migration: remove Dockerfile and use the python-connector-base image | -| 0.2.0 | 2023-10-10 | https://github.com/airbytehq/airbyte/pull/31336 | Migrate to File-based CDK. Add support of CSV, Parquet and Avro files | -| 0.1.0 | 2023-02-17 | https://github.com/airbytehq/airbyte/pull/23222 | Initial release with full-refresh and incremental sync with JSONL files | \ No newline at end of file +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------| +| 0.3.2 | 2024-01-30 | [34661](https://github.com/airbytehq/airbyte/pull/34661) | Pin CDK version until upgrade for compatibility with the Concurrent CDK | +| 0.3.1 | 2024-01-10 | [34084](https://github.com/airbytehq/airbyte/pull/34084) | Fix bug for running check with document file format | +| 0.3.0 | 2023-12-14 | [33411](https://github.com/airbytehq/airbyte/pull/33411) | Bump CDK version to auto-set primary key for document file streams and support raw txt files | +| 0.2.5 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | +| 0.2.4 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Improve document file type parser | +| 0.2.3 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | +| 0.2.2 | 2023-10-30 | [31904](https://github.com/airbytehq/airbyte/pull/31904) | Update CDK to support document file types | +| 0.2.1 | 2023-10-18 | [31543](https://github.com/airbytehq/airbyte/pull/31543) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.2.0 | 2023-10-10 | https://github.com/airbytehq/airbyte/pull/31336 | Migrate to File-based CDK. Add support of CSV, Parquet and Avro files | +| 0.1.0 | 2023-02-17 | https://github.com/airbytehq/airbyte/pull/23222 | Initial release with full-refresh and incremental sync with JSONL files | \ No newline at end of file diff --git a/docs/integrations/sources/gcs.md b/docs/integrations/sources/gcs.md index 5c93e1366c16..58e24a17dcc4 100644 --- a/docs/integrations/sources/gcs.md +++ b/docs/integrations/sources/gcs.md @@ -148,6 +148,7 @@ Leaving this field blank (default option) will disallow escaping. | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------| +| 0.3.5 | 2024-01-30 | [34661](https://github.com/airbytehq/airbyte/pull/34661) | Pin CDK version until upgrade for compatibility with the Concurrent CDK | | 0.3.4 | 2024-01-11 | [34158](https://github.com/airbytehq/airbyte/pull/34158) | Fix issue in stream reader for document file type parser | | 0.3.3 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | | 0.3.2 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Improve document file type parser | diff --git a/docs/integrations/sources/google-drive.md b/docs/integrations/sources/google-drive.md index cc0a92099b4e..ca98dfb3129e 100644 --- a/docs/integrations/sources/google-drive.md +++ b/docs/integrations/sources/google-drive.md @@ -247,7 +247,8 @@ Before parsing each document, the connector exports Google Document files to Doc | Version | Date | Pull Request | Subject | |---------|------------|-----------------------------------------------------------|--------------------------------------------------------------| -| 0.0.6 | 2023-12-16 | [33414](https://github.com/airbytehq/airbyte/pull/33414) | Prepare for airbyte-lib | +| 0.0.7 | 2024-01-30 | [34661](https://github.com/airbytehq/airbyte/pull/34661) | Pin CDK version until upgrade for compatibility with the Concurrent CDK | +| 0.0.6 | 2023-12-16 | [33414](https://github.com/airbytehq/airbyte/pull/33414) | Prepare for airbyte-lib | | 0.0.5 | 2023-12-14 | [33411](https://github.com/airbytehq/airbyte/pull/33411) | Bump CDK version to auto-set primary key for document file streams and support raw txt files | | 0.0.4 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | | 0.0.3 | 2023-11-16 | [31458](https://github.com/airbytehq/airbyte/pull/31458) | Improve folder id input and update document file type parser | diff --git a/docs/integrations/sources/microsoft-onedrive.md b/docs/integrations/sources/microsoft-onedrive.md index 6446c72bcb3b..4697bce4734c 100644 --- a/docs/integrations/sources/microsoft-onedrive.md +++ b/docs/integrations/sources/microsoft-onedrive.md @@ -121,6 +121,7 @@ The connector is restricted by normal Microsoft Graph [requests limitation](http | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------| +| 0.1.4 | 2024-01-30 | [34661](https://github.com/airbytehq/airbyte/pull/34661) | Pin CDK version until upgrade for compatibility with the Concurrent CDK | | 0.1.3 | 2024-01-24 | [34478](https://github.com/airbytehq/airbyte/pull/34478) | Fix OAuth | | 0.1.2 | 2021-12-22 | [33745](https://github.com/airbytehq/airbyte/pull/33745) | Add ql and sl to metadata | | 0.1.1 | 2021-12-15 | [33758](https://github.com/airbytehq/airbyte/pull/33758) | Fix for docs name | From 7f2fddf8f47de7cc5cb24a68fad7ceb9d0655f3a Mon Sep 17 00:00:00 2001 From: Marius Posta Date: Tue, 30 Jan 2024 10:55:11 -0800 Subject: [PATCH 31/96] Fix log4j-slf4j-impl version conflicts (#34669) Co-authored-by: Edward Gao --- airbyte-cdk/java/airbyte-cdk/README.md | 1 + deps.toml | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md index 83d867eadf1e..b8e2addd56a2 100644 --- a/airbyte-cdk/java/airbyte-cdk/README.md +++ b/airbyte-cdk/java/airbyte-cdk/README.md @@ -166,6 +166,7 @@ MavenLocal debugging steps: | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.16.3 | 2024-01-30 | [\#34669](https://github.com/airbytehq/airbyte/pull/34669) | Fix org.apache.logging.log4j:log4j-slf4j-impl version conflicts. | | 0.16.2 | 2024-01-29 | [\#34630](https://github.com/airbytehq/airbyte/pull/34630) | expose NamingTransformer to sub-classes in destinations JdbcSqlGenerator. | | 0.16.1 | 2024-01-29 | [\#34533](https://github.com/airbytehq/airbyte/pull/34533) | Add a safe method to execute DatabaseMetadata's Resultset returning queries. | | 0.16.0 | 2024-01-26 | [\#34573](https://github.com/airbytehq/airbyte/pull/34573) | Untangle Debezium harness dependencies. | diff --git a/deps.toml b/deps.toml index 7c1f15650f72..3ffa1193f4ac 100644 --- a/deps.toml +++ b/deps.toml @@ -81,7 +81,8 @@ kotlinx-cli-jvm = { module = "org.jetbrains.kotlinx:kotlinx-cli-jvm", version = launchdarkly = { module = "com.launchdarkly:launchdarkly-java-server-sdk", version = "6.0.1" } log4j-api = { module = "org.apache.logging.log4j:log4j-api", version.ref = "log4j" } log4j-core = { module = "org.apache.logging.log4j:log4j-core", version.ref = "log4j" } -log4j-impl = { module = "org.apache.logging.log4j:log4j-slf4j2-impl", version.ref = "log4j" } +log4j-slf4j2-impl = { module = "org.apache.logging.log4j:log4j-slf4j2-impl", version.ref = "log4j" } +log4j-slf4j-impl = { module = "org.apache.logging.log4j:log4j-slf4j-impl", version.ref = "log4j" } log4j-over-slf4j = { module = "org.slf4j:log4j-over-slf4j", version.ref = "slf4j" } log4j-web = { module = "org.apache.logging.log4j:log4j-web", version.ref = "log4j" } lombok = { module = "org.projectlombok:lombok", version.ref = "lombok" } @@ -117,7 +118,7 @@ apache = ["apache-commons", "apache-commons-lang"] datadog = ["datadog-trace-api", "datadog-trace-ot"] jackson = ["jackson-databind", "jackson-annotations", "jackson-dataformat", "jackson-datatype"] junit = ["junit-jupiter-api", "junit-jupiter-params", "mockito-junit-jupiter"] -log4j = ["log4j-api", "log4j-core", "log4j-impl", "log4j-web"] +log4j = ["log4j-api", "log4j-core", "log4j-slf4j-impl", "log4j-slf4j2-impl", "log4j-web"] slf4j = ["jul-to-slf4j", "jcl-over-slf4j", "log4j-over-slf4j"] temporal = ["temporal-sdk", "temporal-serviceclient"] From c391e2ecf3481540339013e939a038a70a8ae2f2 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Tue, 30 Jan 2024 11:06:43 -0800 Subject: [PATCH 32/96] Source Faker: Declare primary keys (#34644) --- airbyte-integrations/connectors/source-faker/Dockerfile | 2 +- airbyte-integrations/connectors/source-faker/metadata.yaml | 5 ++++- .../connectors/source-faker/source_faker/streams.py | 6 +++--- docs/integrations/sources/faker-migrations.md | 5 +++++ docs/integrations/sources/faker.md | 1 + 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-faker/Dockerfile b/airbyte-integrations/connectors/source-faker/Dockerfile index 9db110142dbc..3e2d98e54169 100644 --- a/airbyte-integrations/connectors/source-faker/Dockerfile +++ b/airbyte-integrations/connectors/source-faker/Dockerfile @@ -34,5 +34,5 @@ COPY source_faker ./source_faker ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=5.0.2 +LABEL io.airbyte.version=6.0.0 LABEL io.airbyte.name=airbyte/source-faker diff --git a/airbyte-integrations/connectors/source-faker/metadata.yaml b/airbyte-integrations/connectors/source-faker/metadata.yaml index e228708b816e..f91929582dc7 100644 --- a/airbyte-integrations/connectors/source-faker/metadata.yaml +++ b/airbyte-integrations/connectors/source-faker/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: api connectorType: source definitionId: dfd88b22-b603-4c3d-aad7-3701784586b1 - dockerImageTag: 5.0.2 + dockerImageTag: 6.0.0 dockerRepository: airbyte/source-faker documentationUrl: https://docs.airbyte.com/integrations/sources/faker githubIssueLabel: source-faker @@ -30,6 +30,9 @@ data: ID and products.year fields are changing to be integers instead of floats. upgradeDeadline: "2023-08-31" + 6.0.0: + message: Declare 'id' columns as primary keys. + upgradeDeadline: "2024-04-01" resourceRequirements: jobSpecific: - jobType: sync diff --git a/airbyte-integrations/connectors/source-faker/source_faker/streams.py b/airbyte-integrations/connectors/source-faker/source_faker/streams.py index 002866ba7c54..9b015e94851b 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/streams.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/streams.py @@ -15,7 +15,7 @@ class Products(Stream, IncrementalMixin): - primary_key = None + primary_key = "id" cursor_field = "updated_at" def __init__(self, count: int, seed: int, parallelism: int, records_per_slice: int, always_updated: bool, **kwargs): @@ -65,7 +65,7 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: class Users(Stream, IncrementalMixin): - primary_key = None + primary_key = "id" cursor_field = "updated_at" def __init__(self, count: int, seed: int, parallelism: int, records_per_slice: int, always_updated: bool, **kwargs): @@ -125,7 +125,7 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: class Purchases(Stream, IncrementalMixin): - primary_key = None + primary_key = "id" cursor_field = "updated_at" def __init__(self, count: int, seed: int, parallelism: int, records_per_slice: int, always_updated: bool, **kwargs): diff --git a/docs/integrations/sources/faker-migrations.md b/docs/integrations/sources/faker-migrations.md index 46dc3247f93f..7eac0a780b1f 100644 --- a/docs/integrations/sources/faker-migrations.md +++ b/docs/integrations/sources/faker-migrations.md @@ -1,6 +1,11 @@ # Sample Data (Faker) Migration Guide +## Upgrading to 6.0.0 + +All streams (`users`, `products`, and `purchases`) now properly declare `id` as their respective primary keys. Existing sync jobs should still work as expected but you may need to reset your sync and/or update write mode after upgrading to the latest connector version. + ## Upgrading to 5.0.0 + Some columns are narrowing from `number` to `integer`. You may need to force normalization to rebuild your destination tables by manually dropping the SCD and final tables, refreshing the connection schema (skipping the reset), and running a sync. Alternatively, you can just run a reset. ## Upgrading to 4.0.0 diff --git a/docs/integrations/sources/faker.md b/docs/integrations/sources/faker.md index 39a58897e124..7dc02edaebbf 100644 --- a/docs/integrations/sources/faker.md +++ b/docs/integrations/sources/faker.md @@ -101,6 +101,7 @@ None! | Version | Date | Pull Request | Subject | | :------ | :--------- | :-------------------------------------------------------------------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------- | +| 6.0.0 | 2024-01-30 | [34644](https://github.com/airbytehq/airbyte/pull/34644) | Declare 'id' columns as primary keys. | | 5.0.2 | 2024-01-17 | [34344](https://github.com/airbytehq/airbyte/pull/34344) | Ensure unique state messages | | 5.0.1 | 2023-01-08 | [34033](https://github.com/airbytehq/airbyte/pull/34033) | Add standard entrypoints for usage with AirbyteLib | | 5.0.0 | 2023-08-08 | [29213](https://github.com/airbytehq/airbyte/pull/29213) | Change all `*id` fields and `products.year` to be integer | From d13b7c9e4bdc993d3c29ac220a283270fe7f939f Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Tue, 30 Jan 2024 11:42:44 -0800 Subject: [PATCH 33/96] Destination Redshift - Bump CDK version to 0.16.3 (#34680) --- .../destination-redshift/build.gradle | 2 +- .../destination-redshift/metadata.yaml | 2 +- docs/integrations/destinations/redshift.md | 177 +++++++++--------- 3 files changed, 91 insertions(+), 90 deletions(-) diff --git a/airbyte-integrations/connectors/destination-redshift/build.gradle b/airbyte-integrations/connectors/destination-redshift/build.gradle index 857afce64bd3..ab30bd170f23 100644 --- a/airbyte-integrations/connectors/destination-redshift/build.gradle +++ b/airbyte-integrations/connectors/destination-redshift/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.16.1' + cdkVersionRequired = '0.16.3' features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/destination-redshift/metadata.yaml b/airbyte-integrations/connectors/destination-redshift/metadata.yaml index ef63184d6de8..b43e36361d80 100644 --- a/airbyte-integrations/connectors/destination-redshift/metadata.yaml +++ b/airbyte-integrations/connectors/destination-redshift/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: destination definitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc - dockerImageTag: 2.1.4 + dockerImageTag: 2.1.5 dockerRepository: airbyte/destination-redshift documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift githubIssueLabel: destination-redshift diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index b93377f3102e..a90b4d3968a1 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -235,91 +235,92 @@ Each stream will be output into its own raw table in Redshift. Each table will c ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 2.1.4 | 2024-01-29 | [34634](https://github.com/airbytehq/airbyte/pull/34634) | Use lowercase raw schema and table in T+D [CDK changes](https://github.com/airbytehq/airbyte/pull/34533) | -| 2.1.3 | 2024-01-26 | [34544](https://github.com/airbytehq/airbyte/pull/34544) | Proper string-escaping in raw tables | -| 2.1.2 | 2024-01-24 | [34451](https://github.com/airbytehq/airbyte/pull/34451) | Improve logging for unparseable input | -| 2.1.1 | 2024-01-24 | [34458](https://github.com/airbytehq/airbyte/pull/34458) | Improve error reporting | -| 2.1.0 | 2024-01-24 | [34467](https://github.com/airbytehq/airbyte/pull/34467) | Upgrade CDK to 0.14.0 | -| 2.0.0 | 2024-01-23 | [\#34077](https://github.com/airbytehq/airbyte/pull/34077) | Destinations V2 | -| 0.8.0 | 2024-01-18 | [\#34236](https://github.com/airbytehq/airbyte/pull/34236) | Upgrade CDK to 0.13.0 | -| 0.7.15 | 2024-01-11 | [\#34186](https://github.com/airbytehq/airbyte/pull/34186) | Update check method with svv_table_info permission check, fix bug where s3 staging files were not being deleted. | -| 0.7.14 | 2024-01-08 | [\#34014](https://github.com/airbytehq/airbyte/pull/34014) | Update order of options in spec | -| 0.7.13 | 2024-01-05 | [\#33948](https://github.com/airbytehq/airbyte/pull/33948) | Fix NPE when prepare tables fail; Add case sensitive session for super; Bastion heartbeats added | -| 0.7.12 | 2024-01-03 | [\#33924](https://github.com/airbytehq/airbyte/pull/33924) | Add new ap-southeast-3 AWS region | -| 0.7.11 | 2024-01-04 | [\#33730](https://github.com/airbytehq/airbyte/pull/33730) | Internal code structure changes | -| 0.7.10 | 2024-01-04 | [\#33728](https://github.com/airbytehq/airbyte/pull/33728) | Allow users to disable final table creation | -| 0.7.9 | 2024-01-03 | [\#33877](https://github.com/airbytehq/airbyte/pull/33877) | Fix Jooq StackOverflowError | -| 0.7.8 | 2023-12-28 | [\#33788](https://github.com/airbytehq/airbyte/pull/33788) | Thread-safe fix for file part names (s3 staging files) | -| 0.7.7 | 2024-01-04 | [\#33728](https://github.com/airbytehq/airbyte/pull/33728) | Add option to only type and dedupe at the end of the sync | -| 0.7.6 | 2023-12-20 | [\#33704](https://github.com/airbytehq/airbyte/pull/33704) | Only run T+D on a stream if it had any records during the sync | -| 0.7.5 | 2023-12-18 | [\#33124](https://github.com/airbytehq/airbyte/pull/33124) | Make Schema Creation Separate from Table Creation | -| 0.7.4 | 2023-12-13 | [\#33369](https://github.com/airbytehq/airbyte/pull/33369) | Use jdbc common sql implementation | -| 0.7.3 | 2023-12-12 | [\#33367](https://github.com/airbytehq/airbyte/pull/33367) | DV2: fix migration logic | -| 0.7.2 | 2023-12-11 | [\#33335](https://github.com/airbytehq/airbyte/pull/33335) | DV2: improve data type mapping | -| 0.7.1 | 2023-12-11 | [\#33307](https://github.com/airbytehq/airbyte/pull/33307) | ~DV2: improve data type mapping~ No changes | -| 0.7.0 | 2023-12-05 | [\#32326](https://github.com/airbytehq/airbyte/pull/32326) | Opt in beta for v2 destination | -| 0.6.11 | 2023-11-29 | [\#32888](https://github.com/airbytehq/airbyte/pull/32888) | Use the new async framework. | -| 0.6.10 | 2023-11-06 | [\#32193](https://github.com/airbytehq/airbyte/pull/32193) | Adopt java CDK version 0.4.1. | -| 0.6.9 | 2023-10-10 | [\#31083](https://github.com/airbytehq/airbyte/pull/31083) | Fix precision of numeric values in async destinations | -| 0.6.8 | 2023-10-10 | [\#31218](https://github.com/airbytehq/airbyte/pull/31218) | Clarify configuration groups | -| 0.6.7 | 2023-10-06 | [\#31153](https://github.com/airbytehq/airbyte/pull/31153) | Increase jvm GC retries | -| 0.6.6 | 2023-10-06 | [\#31129](https://github.com/airbytehq/airbyte/pull/31129) | Reduce async buffer size | -| 0.6.5 | 2023-08-18 | [\#28619](https://github.com/airbytehq/airbyte/pull/29640) | Fix duplicate staging object names in concurrent environment (e.g. async) | -| 0.6.4 | 2023-08-10 | [\#28619](https://github.com/airbytehq/airbyte/pull/28619) | Use async method for staging | -| 0.6.3 | 2023-08-07 | [\#29188](https://github.com/airbytehq/airbyte/pull/29188) | Internal code refactoring | -| 0.6.2 | 2023-07-24 | [\#28618](https://github.com/airbytehq/airbyte/pull/28618) | Add hooks in preparation for destinations v2 implementation | -| 0.6.1 | 2023-07-14 | [\#28345](https://github.com/airbytehq/airbyte/pull/28345) | Increment patch to trigger a rebuild | -| 0.6.0 | 2023-06-27 | [\#27993](https://github.com/airbytehq/airbyte/pull/27993) | destination-redshift will fail syncs if records or properties are too large, rather than silently skipping records and succeeding | -| 0.5.0 | 2023-06-27 | [\#27781](https://github.com/airbytehq/airbyte/pull/27781) | License Update: Elv2 | -| 0.4.9 | 2023-06-21 | [\#27555](https://github.com/airbytehq/airbyte/pull/27555) | Reduce image size | -| 0.4.8 | 2023-05-17 | [\#26165](https://github.com/airbytehq/airbyte/pull/26165) | Internal code change for future development (install normalization packages inside connector) | -| 0.4.7 | 2023-05-01 | [\#25698](https://github.com/airbytehq/airbyte/pull/25698) | Remove old VARCHAR to SUPER migration Java functionality | -| 0.4.6 | 2023-04-27 | [\#25346](https://github.com/airbytehq/airbyte/pull/25346) | Internal code cleanup | -| 0.4.5 | 2023-03-30 | [\#24736](https://github.com/airbytehq/airbyte/pull/24736) | Improve behavior when throttled by AWS API | -| 0.4.4 | 2023-03-29 | [\#24671](https://github.com/airbytehq/airbyte/pull/24671) | Fail faster in certain error cases | -| 0.4.3 | 2023-03-17 | [\#23788](https://github.com/airbytehq/airbyte/pull/23788) | S3-Parquet: added handler to process null values in arrays | -| 0.4.2 | 2023-03-10 | [\#23931](https://github.com/airbytehq/airbyte/pull/23931) | Added support for periodic buffer flush | -| 0.4.1 | 2023-03-10 | [\#23466](https://github.com/airbytehq/airbyte/pull/23466) | Changed S3 Avro type from Int to Long | -| 0.4.0 | 2023-02-28 | [\#23523](https://github.com/airbytehq/airbyte/pull/23523) | Add SSH Bastion Host configuration options | -| 0.3.56 | 2023-01-26 | [\#21890](https://github.com/airbytehq/airbyte/pull/21890) | Fixed configurable parameter for number of file buffers | -| 0.3.55 | 2023-01-26 | [\#20631](https://github.com/airbytehq/airbyte/pull/20631) | Added support for destination checkpointing with staging | -| 0.3.54 | 2023-01-18 | [\#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions | -| 0.3.53 | 2023-01-03 | [\#17273](https://github.com/airbytehq/airbyte/pull/17273) | Flatten JSON arrays to fix maximum size check for SUPER field | -| 0.3.52 | 2022-12-30 | [\#20879](https://github.com/airbytehq/airbyte/pull/20879) | Added configurable parameter for number of file buffers (⛔ this version has a bug and will not work; use `0.3.56` instead) | -| 0.3.51 | 2022-10-26 | [\#18434](https://github.com/airbytehq/airbyte/pull/18434) | Fix empty S3 bucket path handling | -| 0.3.50 | 2022-09-14 | [\#15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage | -| 0.3.49 | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243) | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields) | -| 0.3.48 | 2022-09-01 | | Added JDBC URL params | -| 0.3.47 | 2022-07-15 | [\#14494](https://github.com/airbytehq/airbyte/pull/14494) | Make S3 output filename configurable. | -| 0.3.46 | 2022-06-27 | [\#14190](https://github.com/airbytehq/airbyte/pull/13916) | Correctly cleanup S3 bucket when using a configured bucket path for S3 staging operations. | -| 0.3.45 | 2022-06-25 | [\#13916](https://github.com/airbytehq/airbyte/pull/13916) | Use the configured bucket path for S3 staging operations. | -| 0.3.44 | 2022-06-24 | [\#14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging | -| 0.3.43 | 2022-06-24 | [\#13690](https://github.com/airbytehq/airbyte/pull/13690) | Improved discovery for NOT SUPER column | -| 0.3.42 | 2022-06-21 | [\#14013](https://github.com/airbytehq/airbyte/pull/14013) | Add an option to use encryption with staging in Redshift Destination | -| 0.3.40 | 2022-06-17 | [\#13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART\*SIZE_MB fields from connectors based on StreamTransferManager | -| 0.3.39 | 2022-06-02 | [\#13415](https://github.com/airbytehq/airbyte/pull/13415) | Add dropdown to select Uploading Method.
**PLEASE NOTICE**: After this update your **uploading method** will be set to **Standard**, you will need to reconfigure the method to use **S3 Staging** again. | -| 0.3.37 | 2022-05-23 | [\#13090](https://github.com/airbytehq/airbyte/pull/13090) | Removed redshiftDataTmpTableMode. Some refactoring. | -| 0.3.36 | 2022-05-23 | [\#12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | -| 0.3.35 | 2022-05-18 | [\#12940](https://github.com/airbytehq/airbyte/pull/12940) | Fixed maximum record size for SUPER type | -| 0.3.34 | 2022-05-16 | [\#12869](https://github.com/airbytehq/airbyte/pull/12869) | Fixed NPE in S3 staging check | -| 0.3.33 | 2022-05-04 | [\#12601](https://github.com/airbytehq/airbyte/pull/12601) | Apply buffering strategy for S3 staging | -| 0.3.32 | 2022-04-20 | [\#12085](https://github.com/airbytehq/airbyte/pull/12085) | Fixed bug with switching between INSERT and COPY config | -| 0.3.31 | 2022-04-19 | [\#12064](https://github.com/airbytehq/airbyte/pull/12064) | Added option to support SUPER datatype in \_airbyte_raw\*\*\* table | -| 0.3.29 | 2022-04-05 | [\#11729](https://github.com/airbytehq/airbyte/pull/11729) | Fixed bug with dashes in schema name | -| 0.3.28 | 2022-03-18 | [\#11254](https://github.com/airbytehq/airbyte/pull/11254) | Fixed missing records during S3 staging | -| 0.3.27 | 2022-02-25 | [\#10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | -| 0.3.25 | 2022-02-14 | [\#9920](https://github.com/airbytehq/airbyte/pull/9920) | Updated the size of staging files for S3 staging. Also, added closure of S3 writers to staging files when data has been written to an staging file. | -| 0.3.24 | 2022-02-14 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.3.23 | 2021-12-16 | [\#8855](https://github.com/airbytehq/airbyte/pull/8855) | Add `purgeStagingData` option to enable/disable deleting the staging data | -| 0.3.22 | 2021-12-15 | [\#8607](https://github.com/airbytehq/airbyte/pull/8607) | Accept a path for the staging data | -| 0.3.21 | 2021-12-10 | [\#8562](https://github.com/airbytehq/airbyte/pull/8562) | Moving classes around for better dependency management | -| 0.3.20 | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | -| 0.3.19 | 2021-10-21 | [\#7234](https://github.com/airbytehq/airbyte/pull/7234) | Allow SSL traffic only | -| 0.3.17 | 2021-10-12 | [\#6965](https://github.com/airbytehq/airbyte/pull/6965) | Added SSL Support | -| 0.3.16 | 2021-10-11 | [\#6949](https://github.com/airbytehq/airbyte/pull/6949) | Each stream was split into files of 10,000 records each for copying using S3 or GCS | -| 0.3.14 | 2021-10-08 | [\#5924](https://github.com/airbytehq/airbyte/pull/5924) | Fixed AWS S3 Staging COPY is writing records from different table in the same raw table | -| 0.3.13 | 2021-09-02 | [\#5745](https://github.com/airbytehq/airbyte/pull/5745) | Disable STATUPDATE flag when using S3 staging to speed up performance | -| 0.3.12 | 2021-07-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555) | Enable partial checkpointing for halfway syncs | -| 0.3.11 | 2021-07-20 | [\#4874](https://github.com/airbytehq/airbyte/pull/4874) | allow `additionalProperties` in connector spec | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.1.5 | 2024-01-30 | [\#34680](https://github.com/airbytehq/airbyte/pull/34680) | Update to CDK version 0.16.3 | +| 2.1.4 | 2024-01-29 | [\#34634](https://github.com/airbytehq/airbyte/pull/34634) | Use lowercase raw schema and table in T+D [CDK changes](https://github.com/airbytehq/airbyte/pull/34533) | +| 2.1.3 | 2024-01-26 | [\#34544](https://github.com/airbytehq/airbyte/pull/34544) | Proper string-escaping in raw tables | +| 2.1.2 | 2024-01-24 | [\#34451](https://github.com/airbytehq/airbyte/pull/34451) | Improve logging for unparseable input | +| 2.1.1 | 2024-01-24 | [\#34458](https://github.com/airbytehq/airbyte/pull/34458) | Improve error reporting | +| 2.1.0 | 2024-01-24 | [\#34467](https://github.com/airbytehq/airbyte/pull/34467) | Upgrade CDK to 0.14.0 | +| 2.0.0 | 2024-01-23 | [\#34077](https://github.com/airbytehq/airbyte/pull/34077) | Destinations V2 | +| 0.8.0 | 2024-01-18 | [\#34236](https://github.com/airbytehq/airbyte/pull/34236) | Upgrade CDK to 0.13.0 | +| 0.7.15 | 2024-01-11 | [\#34186](https://github.com/airbytehq/airbyte/pull/34186) | Update check method with svv_table_info permission check, fix bug where s3 staging files were not being deleted. | +| 0.7.14 | 2024-01-08 | [\#34014](https://github.com/airbytehq/airbyte/pull/34014) | Update order of options in spec | +| 0.7.13 | 2024-01-05 | [\#33948](https://github.com/airbytehq/airbyte/pull/33948) | Fix NPE when prepare tables fail; Add case sensitive session for super; Bastion heartbeats added | +| 0.7.12 | 2024-01-03 | [\#33924](https://github.com/airbytehq/airbyte/pull/33924) | Add new ap-southeast-3 AWS region | +| 0.7.11 | 2024-01-04 | [\#33730](https://github.com/airbytehq/airbyte/pull/33730) | Internal code structure changes | +| 0.7.10 | 2024-01-04 | [\#33728](https://github.com/airbytehq/airbyte/pull/33728) | Allow users to disable final table creation | +| 0.7.9 | 2024-01-03 | [\#33877](https://github.com/airbytehq/airbyte/pull/33877) | Fix Jooq StackOverflowError | +| 0.7.8 | 2023-12-28 | [\#33788](https://github.com/airbytehq/airbyte/pull/33788) | Thread-safe fix for file part names (s3 staging files) | +| 0.7.7 | 2024-01-04 | [\#33728](https://github.com/airbytehq/airbyte/pull/33728) | Add option to only type and dedupe at the end of the sync | +| 0.7.6 | 2023-12-20 | [\#33704](https://github.com/airbytehq/airbyte/pull/33704) | Only run T+D on a stream if it had any records during the sync | +| 0.7.5 | 2023-12-18 | [\#33124](https://github.com/airbytehq/airbyte/pull/33124) | Make Schema Creation Separate from Table Creation | +| 0.7.4 | 2023-12-13 | [\#33369](https://github.com/airbytehq/airbyte/pull/33369) | Use jdbc common sql implementation | +| 0.7.3 | 2023-12-12 | [\#33367](https://github.com/airbytehq/airbyte/pull/33367) | DV2: fix migration logic | +| 0.7.2 | 2023-12-11 | [\#33335](https://github.com/airbytehq/airbyte/pull/33335) | DV2: improve data type mapping | +| 0.7.1 | 2023-12-11 | [\#33307](https://github.com/airbytehq/airbyte/pull/33307) | ~DV2: improve data type mapping~ No changes | +| 0.7.0 | 2023-12-05 | [\#32326](https://github.com/airbytehq/airbyte/pull/32326) | Opt in beta for v2 destination | +| 0.6.11 | 2023-11-29 | [\#32888](https://github.com/airbytehq/airbyte/pull/32888) | Use the new async framework. | +| 0.6.10 | 2023-11-06 | [\#32193](https://github.com/airbytehq/airbyte/pull/32193) | Adopt java CDK version 0.4.1. | +| 0.6.9 | 2023-10-10 | [\#31083](https://github.com/airbytehq/airbyte/pull/31083) | Fix precision of numeric values in async destinations | +| 0.6.8 | 2023-10-10 | [\#31218](https://github.com/airbytehq/airbyte/pull/31218) | Clarify configuration groups | +| 0.6.7 | 2023-10-06 | [\#31153](https://github.com/airbytehq/airbyte/pull/31153) | Increase jvm GC retries | +| 0.6.6 | 2023-10-06 | [\#31129](https://github.com/airbytehq/airbyte/pull/31129) | Reduce async buffer size | +| 0.6.5 | 2023-08-18 | [\#28619](https://github.com/airbytehq/airbyte/pull/29640) | Fix duplicate staging object names in concurrent environment (e.g. async) | +| 0.6.4 | 2023-08-10 | [\#28619](https://github.com/airbytehq/airbyte/pull/28619) | Use async method for staging | +| 0.6.3 | 2023-08-07 | [\#29188](https://github.com/airbytehq/airbyte/pull/29188) | Internal code refactoring | +| 0.6.2 | 2023-07-24 | [\#28618](https://github.com/airbytehq/airbyte/pull/28618) | Add hooks in preparation for destinations v2 implementation | +| 0.6.1 | 2023-07-14 | [\#28345](https://github.com/airbytehq/airbyte/pull/28345) | Increment patch to trigger a rebuild | +| 0.6.0 | 2023-06-27 | [\#27993](https://github.com/airbytehq/airbyte/pull/27993) | destination-redshift will fail syncs if records or properties are too large, rather than silently skipping records and succeeding | +| 0.5.0 | 2023-06-27 | [\#27781](https://github.com/airbytehq/airbyte/pull/27781) | License Update: Elv2 | +| 0.4.9 | 2023-06-21 | [\#27555](https://github.com/airbytehq/airbyte/pull/27555) | Reduce image size | +| 0.4.8 | 2023-05-17 | [\#26165](https://github.com/airbytehq/airbyte/pull/26165) | Internal code change for future development (install normalization packages inside connector) | +| 0.4.7 | 2023-05-01 | [\#25698](https://github.com/airbytehq/airbyte/pull/25698) | Remove old VARCHAR to SUPER migration Java functionality | +| 0.4.6 | 2023-04-27 | [\#25346](https://github.com/airbytehq/airbyte/pull/25346) | Internal code cleanup | +| 0.4.5 | 2023-03-30 | [\#24736](https://github.com/airbytehq/airbyte/pull/24736) | Improve behavior when throttled by AWS API | +| 0.4.4 | 2023-03-29 | [\#24671](https://github.com/airbytehq/airbyte/pull/24671) | Fail faster in certain error cases | +| 0.4.3 | 2023-03-17 | [\#23788](https://github.com/airbytehq/airbyte/pull/23788) | S3-Parquet: added handler to process null values in arrays | +| 0.4.2 | 2023-03-10 | [\#23931](https://github.com/airbytehq/airbyte/pull/23931) | Added support for periodic buffer flush | +| 0.4.1 | 2023-03-10 | [\#23466](https://github.com/airbytehq/airbyte/pull/23466) | Changed S3 Avro type from Int to Long | +| 0.4.0 | 2023-02-28 | [\#23523](https://github.com/airbytehq/airbyte/pull/23523) | Add SSH Bastion Host configuration options | +| 0.3.56 | 2023-01-26 | [\#21890](https://github.com/airbytehq/airbyte/pull/21890) | Fixed configurable parameter for number of file buffers | +| 0.3.55 | 2023-01-26 | [\#20631](https://github.com/airbytehq/airbyte/pull/20631) | Added support for destination checkpointing with staging | +| 0.3.54 | 2023-01-18 | [\#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions | +| 0.3.53 | 2023-01-03 | [\#17273](https://github.com/airbytehq/airbyte/pull/17273) | Flatten JSON arrays to fix maximum size check for SUPER field | +| 0.3.52 | 2022-12-30 | [\#20879](https://github.com/airbytehq/airbyte/pull/20879) | Added configurable parameter for number of file buffers (⛔ this version has a bug and will not work; use `0.3.56` instead) | +| 0.3.51 | 2022-10-26 | [\#18434](https://github.com/airbytehq/airbyte/pull/18434) | Fix empty S3 bucket path handling | +| 0.3.50 | 2022-09-14 | [\#15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage | +| 0.3.49 | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243) | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields) | +| 0.3.48 | 2022-09-01 | | Added JDBC URL params | +| 0.3.47 | 2022-07-15 | [\#14494](https://github.com/airbytehq/airbyte/pull/14494) | Make S3 output filename configurable. | +| 0.3.46 | 2022-06-27 | [\#14190](https://github.com/airbytehq/airbyte/pull/13916) | Correctly cleanup S3 bucket when using a configured bucket path for S3 staging operations. | +| 0.3.45 | 2022-06-25 | [\#13916](https://github.com/airbytehq/airbyte/pull/13916) | Use the configured bucket path for S3 staging operations. | +| 0.3.44 | 2022-06-24 | [\#14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging | +| 0.3.43 | 2022-06-24 | [\#13690](https://github.com/airbytehq/airbyte/pull/13690) | Improved discovery for NOT SUPER column | +| 0.3.42 | 2022-06-21 | [\#14013](https://github.com/airbytehq/airbyte/pull/14013) | Add an option to use encryption with staging in Redshift Destination | +| 0.3.40 | 2022-06-17 | [\#13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART\*SIZE_MB fields from connectors based on StreamTransferManager | +| 0.3.39 | 2022-06-02 | [\#13415](https://github.com/airbytehq/airbyte/pull/13415) | Add dropdown to select Uploading Method.
**PLEASE NOTICE**: After this update your **uploading method** will be set to **Standard**, you will need to reconfigure the method to use **S3 Staging** again. | +| 0.3.37 | 2022-05-23 | [\#13090](https://github.com/airbytehq/airbyte/pull/13090) | Removed redshiftDataTmpTableMode. Some refactoring. | +| 0.3.36 | 2022-05-23 | [\#12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | +| 0.3.35 | 2022-05-18 | [\#12940](https://github.com/airbytehq/airbyte/pull/12940) | Fixed maximum record size for SUPER type | +| 0.3.34 | 2022-05-16 | [\#12869](https://github.com/airbytehq/airbyte/pull/12869) | Fixed NPE in S3 staging check | +| 0.3.33 | 2022-05-04 | [\#12601](https://github.com/airbytehq/airbyte/pull/12601) | Apply buffering strategy for S3 staging | +| 0.3.32 | 2022-04-20 | [\#12085](https://github.com/airbytehq/airbyte/pull/12085) | Fixed bug with switching between INSERT and COPY config | +| 0.3.31 | 2022-04-19 | [\#12064](https://github.com/airbytehq/airbyte/pull/12064) | Added option to support SUPER datatype in \_airbyte_raw\*\*\* table | +| 0.3.29 | 2022-04-05 | [\#11729](https://github.com/airbytehq/airbyte/pull/11729) | Fixed bug with dashes in schema name | +| 0.3.28 | 2022-03-18 | [\#11254](https://github.com/airbytehq/airbyte/pull/11254) | Fixed missing records during S3 staging | +| 0.3.27 | 2022-02-25 | [\#10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | +| 0.3.25 | 2022-02-14 | [\#9920](https://github.com/airbytehq/airbyte/pull/9920) | Updated the size of staging files for S3 staging. Also, added closure of S3 writers to staging files when data has been written to an staging file. | +| 0.3.24 | 2022-02-14 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.3.23 | 2021-12-16 | [\#8855](https://github.com/airbytehq/airbyte/pull/8855) | Add `purgeStagingData` option to enable/disable deleting the staging data | +| 0.3.22 | 2021-12-15 | [\#8607](https://github.com/airbytehq/airbyte/pull/8607) | Accept a path for the staging data | +| 0.3.21 | 2021-12-10 | [\#8562](https://github.com/airbytehq/airbyte/pull/8562) | Moving classes around for better dependency management | +| 0.3.20 | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | +| 0.3.19 | 2021-10-21 | [\#7234](https://github.com/airbytehq/airbyte/pull/7234) | Allow SSL traffic only | +| 0.3.17 | 2021-10-12 | [\#6965](https://github.com/airbytehq/airbyte/pull/6965) | Added SSL Support | +| 0.3.16 | 2021-10-11 | [\#6949](https://github.com/airbytehq/airbyte/pull/6949) | Each stream was split into files of 10,000 records each for copying using S3 or GCS | +| 0.3.14 | 2021-10-08 | [\#5924](https://github.com/airbytehq/airbyte/pull/5924) | Fixed AWS S3 Staging COPY is writing records from different table in the same raw table | +| 0.3.13 | 2021-09-02 | [\#5745](https://github.com/airbytehq/airbyte/pull/5745) | Disable STATUPDATE flag when using S3 staging to speed up performance | +| 0.3.12 | 2021-07-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555) | Enable partial checkpointing for halfway syncs | +| 0.3.11 | 2021-07-20 | [\#4874](https://github.com/airbytehq/airbyte/pull/4874) | allow `additionalProperties` in connector spec | From 3bd0acd739b7127465dec96b5cac1dcefeb26489 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Tue, 30 Jan 2024 19:43:34 -0300 Subject: [PATCH 34/96] Destination Teradata: make connector avaialble on Airbyte Cloud (#28667) Co-authored-by: SatishChGit Co-authored-by: evantahler --- .../destination-teradata/metadata.yaml | 5 ++--- .../teradata/TeradataDestination.java | 17 +++++++++++------ .../teradata/TeradataSqlOperations.java | 6 +++--- .../TeradataDestinationAcceptanceTest.java | 6 ++++++ docs/integrations/destinations/teradata.md | 16 +++++++++------- 5 files changed, 31 insertions(+), 19 deletions(-) diff --git a/airbyte-integrations/connectors/destination-teradata/metadata.yaml b/airbyte-integrations/connectors/destination-teradata/metadata.yaml index 91738294ff4e..de975748835f 100644 --- a/airbyte-integrations/connectors/destination-teradata/metadata.yaml +++ b/airbyte-integrations/connectors/destination-teradata/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 58e6f9da-904e-11ed-a1eb-0242ac120002 - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.5 dockerRepository: airbyte/destination-teradata githubIssueLabel: destination-teradata icon: teradata.svg @@ -10,12 +10,11 @@ data: name: Teradata Vantage registries: cloud: - enabled: false + enabled: true oss: enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/teradata - supportsDbt: true tags: - language:java ab_internal: diff --git a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java index 37fd84a973a6..55aa93c237b4 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java +++ b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java @@ -49,6 +49,10 @@ public class TeradataDestination extends AbstractJdbcDestination implements Dest protected static final String CA_CERT_KEY = "ssl_ca_certificate"; + protected static final String ENCRYPTDATA = "ENCRYPTDATA"; + + protected static final String ENCRYPTDATA_ON = "ON"; + public static void main(String[] args) throws Exception { new IntegrationRunner(new TeradataDestination()).run(args); } @@ -57,6 +61,12 @@ public TeradataDestination() { super(DRIVER_CLASS, new StandardNameTransformer(), new TeradataSqlOperations()); } + private static void createCertificateFile(String fileName, String fileValue) throws IOException { + try (final PrintWriter out = new PrintWriter(fileName, StandardCharsets.UTF_8)) { + out.print(fileValue); + } + } + @Override protected Map getDefaultConnectionProperties(final JsonNode config) { final Map additionalParameters = new HashMap<>(); @@ -69,15 +79,10 @@ protected Map getDefaultConnectionProperties(final JsonNode conf additionalParameters.put(PARAM_SSLMODE, REQUIRE); } } + additionalParameters.put(ENCRYPTDATA, ENCRYPTDATA_ON); return additionalParameters; } - private static void createCertificateFile(String fileName, String fileValue) throws IOException { - try (final PrintWriter out = new PrintWriter(fileName, StandardCharsets.UTF_8)) { - out.print(fileValue); - } - } - private Map obtainConnectionOptions(final JsonNode encryption) { final Map additionalParameters = new HashMap<>(); if (!encryption.isNull()) { diff --git a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java index 85cf7dc27d53..55522de02b66 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java +++ b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java @@ -107,10 +107,10 @@ public void createTableIfNotExists(final JdbcDatabase database, final String sch @Override public String createTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { return String.format( - "CREATE SET TABLE %s.%s, FALLBACK ( \n" + "%s VARCHAR(256), \n" + "%s JSON, \n" + "%s TIMESTAMP(6) \n" - + ");\n", + "CREATE SET TABLE %s.%s, FALLBACK ( %s VARCHAR(256), %s JSON, %s TIMESTAMP(6)) " + + " UNIQUE PRIMARY INDEX (%s) ", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_DATA, - JavaBaseConstants.COLUMN_NAME_EMITTED_AT); + JavaBaseConstants.COLUMN_NAME_EMITTED_AT, JavaBaseConstants.COLUMN_NAME_AB_ID); } @Override diff --git a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java index c3fa9274ad98..ea6969b8c440 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java @@ -179,6 +179,12 @@ public void testSecondSync() { // overrides test in coming releases } + @Override + @Test + public void testCustomDbtTransformations() throws Exception { + // overrides test in coming releases + } + protected DataSource getDataSource(final JsonNode config) { final JsonNode jdbcConfig = destination.toJdbcConfig(config); return DataSourceFactory.create(jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText(), diff --git a/docs/integrations/destinations/teradata.md b/docs/integrations/destinations/teradata.md index 74f2b89fd598..8f6bfd22c0f2 100644 --- a/docs/integrations/destinations/teradata.md +++ b/docs/integrations/destinations/teradata.md @@ -26,7 +26,7 @@ You'll need the following information to configure the Teradata destination: Each stream will be output into its own table in Teradata. Each table will contain 3 columns: -- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in Teradata is `VARCHAR(256)`. +- `_airbyte_ab_id`: a unique uuid assigned by Airbyte to each event that is processed. This is the primary index column. The column type in Teradata is `VARCHAR(256)`. - `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in Teradata is `TIMESTAMP(6)`. - `_airbyte_data`: a json blob representing with the event data. The column type in Teradata is `JSON`. @@ -84,9 +84,11 @@ You can also use a pre-existing user but we highly recommend creating a dedicate ## CHANGELOG -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :---------------------------------------------- | :------------------------------- | -| 0.1.3 | 2023-08-17 | https://github.com/airbytehq/airbyte/pull/30740 | Enable custom DBT transformation | -| 0.1.2 | 2023-08-09 | https://github.com/airbytehq/airbyte/pull/29174 | Small internal refactor | -| 0.1.1 | 2023-03-03 | https://github.com/airbytehq/airbyte/pull/21760 | Added SSL support | -| 0.1.0 | 2022-12-13 | https://github.com/airbytehq/airbyte/pull/20428 | New Destination Teradata Vantage | +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :---------------------------------------------- |:--------------------------------------------------------| +| 0.1.5 | 2024-01-12 | https://github.com/airbytehq/airbyte/pull/33872 | Added Primary Index on _airbyte_ab_id to fix NoPI issue | +| 0.1.4 | 2023-12-04 | https://github.com/airbytehq/airbyte/pull/28667 | Make connector available on Airbyte Cloud | +| 0.1.3 | 2023-08-17 | https://github.com/airbytehq/airbyte/pull/30740 | Enable custom DBT transformation | +| 0.1.2 | 2023-08-09 | https://github.com/airbytehq/airbyte/pull/29174 | Small internal refactor | +| 0.1.1 | 2023-03-03 | https://github.com/airbytehq/airbyte/pull/21760 | Added SSL support | +| 0.1.0 | 2022-12-13 | https://github.com/airbytehq/airbyte/pull/20428 | New Destination Teradata Vantage | \ No newline at end of file From 73cda9aca02fa5d51e23e3ccdbbe7c07b4744956 Mon Sep 17 00:00:00 2001 From: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Date: Tue, 30 Jan 2024 15:58:57 -0800 Subject: [PATCH 35/96] Support resuming initial snapshot when id type is String, Int, Long (#34641) --- .../source-mongodb-v2/metadata.yaml | 2 +- .../mongodb/InitialSnapshotHandler.java | 13 +++- .../mongodb/InitialSnapshotHandlerTest.java | 60 ++++++++++++++++++- docs/integrations/sources/mongodb-v2.md | 3 +- 4 files changed, 72 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml b/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml index 0f4f93f719b8..469a031351f2 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml +++ b/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: source definitionId: b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e - dockerImageTag: 1.2.4 + dockerImageTag: 1.2.5 dockerRepository: airbyte/source-mongodb-v2 documentationUrl: https://docs.airbyte.com/integrations/sources/mongodb-v2 githubIssueLabel: source-mongodb-v2 diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandler.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandler.java index 5d7d9ad72587..f9863945d8fd 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandler.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandler.java @@ -27,6 +27,10 @@ import java.util.List; import java.util.Optional; import org.bson.BsonDocument; +import org.bson.BsonInt32; +import org.bson.BsonInt64; +import org.bson.BsonObjectId; +import org.bson.BsonString; import org.bson.Document; import org.bson.conversions.Bson; import org.bson.types.ObjectId; @@ -86,8 +90,13 @@ public List> getIterators( // "where _id > [last saved state] order by _id ASC". // If no state exists, it will create a query akin to "where 1=1 order by _id ASC" final Bson filter = existingState - // TODO add type support here when we add support for _id fields that are not ObjectId types - .map(state -> Filters.gt(MongoConstants.ID_FIELD, new ObjectId(state.id()))) + .map(state -> Filters.gt(MongoConstants.ID_FIELD, + switch (state.idType()) { + case STRING -> new BsonString(state.id()); + case OBJECT_ID -> new BsonObjectId(new ObjectId(state.id())); + case INT -> new BsonInt32(Integer.parseInt(state.id())); + case LONG -> new BsonInt64(Long.parseLong(state.id())); + })) // if nothing was found, return a new BsonDocument .orElseGet(BsonDocument::new); diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandlerTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandlerTest.java index 919903c76575..9ece697ec8fa 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandlerTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandlerTest.java @@ -56,9 +56,11 @@ class InitialSnapshotHandlerTest { private static final String COLLECTION3 = "collection3"; private static final String OBJECT_ID1_STRING = "64c0029d95ad260d69ef28a1"; + private static final String OBJECT_ID2_STRING = "64c0029d95ad260d69ef28a2"; + private static final String OBJECT_ID3_STRING = "64c0029d95ad260d69ef28a3"; private static final ObjectId OBJECT_ID1 = new ObjectId(OBJECT_ID1_STRING); - private static final ObjectId OBJECT_ID2 = new ObjectId("64c0029d95ad260d69ef28a2"); - private static final ObjectId OBJECT_ID3 = new ObjectId("64c0029d95ad260d69ef28a3"); + private static final ObjectId OBJECT_ID2 = new ObjectId(OBJECT_ID2_STRING); + private static final ObjectId OBJECT_ID3 = new ObjectId(OBJECT_ID3_STRING); private static final ObjectId OBJECT_ID4 = new ObjectId("64c0029d95ad260d69ef28a4"); private static final ObjectId OBJECT_ID5 = new ObjectId("64c0029d95ad260d69ef28a5"); private static final ObjectId OBJECT_ID6 = new ObjectId("64c0029d95ad260d69ef28a6"); @@ -332,4 +334,58 @@ void testGetIteratorsWithOneEmptyCollection() { assertFalse(collection2.hasNext()); } + @Test + void testGetIteratorsWithInitialStateNonDefaultIdType() { + insertDocuments(COLLECTION1, List.of( + new Document(Map.of( + CURSOR_FIELD, OBJECT_ID1_STRING, + NAME_FIELD, NAME1)), + new Document(Map.of( + CURSOR_FIELD, OBJECT_ID2_STRING, + NAME_FIELD, NAME2)))); + + insertDocuments(COLLECTION2, List.of( + new Document(Map.of( + CURSOR_FIELD, OBJECT_ID3_STRING, + NAME_FIELD, NAME3)))); + + final InitialSnapshotHandler initialSnapshotHandler = new InitialSnapshotHandler(); + final MongoDbStateManager stateManager = mock(MongoDbStateManager.class); + when(stateManager.getStreamState(COLLECTION1, NAMESPACE)) + .thenReturn(Optional.of(new MongoDbStreamState(OBJECT_ID1_STRING, null, IdType.STRING))); + final List> iterators = + initialSnapshotHandler.getIterators(STREAMS, stateManager, mongoClient.getDatabase(DB_NAME), null, Instant.now(), + MongoConstants.CHECKPOINT_INTERVAL, true); + + assertEquals(iterators.size(), 2, "Only two streams are configured as incremental, full refresh streams should be ignored"); + + final AutoCloseableIterator collection1 = iterators.get(0); + final AutoCloseableIterator collection2 = iterators.get(1); + + // collection1, first document should be skipped + final AirbyteMessage collection1StreamMessage1 = collection1.next(); + assertEquals(Type.RECORD, collection1StreamMessage1.getType()); + assertEquals(COLLECTION1, collection1StreamMessage1.getRecord().getStream()); + assertEquals(OBJECT_ID2.toString(), collection1StreamMessage1.getRecord().getData().get(CURSOR_FIELD).asText()); + assertEquals(NAME2, collection1StreamMessage1.getRecord().getData().get(NAME_FIELD).asText()); + assertConfiguredFieldsEqualsRecordDataFields(Set.of(CURSOR_FIELD, NAME_FIELD), collection1StreamMessage1.getRecord().getData()); + + final AirbyteMessage collection1SateMessage = collection1.next(); + assertEquals(Type.STATE, collection1SateMessage.getType(), "State message is expected after all records in a stream are emitted"); + + assertFalse(collection1.hasNext()); + + // collection2, no documents should be skipped + final AirbyteMessage collection2StreamMessage1 = collection2.next(); + assertEquals(Type.RECORD, collection2StreamMessage1.getType()); + assertEquals(COLLECTION2, collection2StreamMessage1.getRecord().getStream()); + assertEquals(OBJECT_ID3.toString(), collection2StreamMessage1.getRecord().getData().get(CURSOR_FIELD).asText()); + assertConfiguredFieldsEqualsRecordDataFields(Set.of(CURSOR_FIELD), collection2StreamMessage1.getRecord().getData()); + + final AirbyteMessage collection2SateMessage = collection2.next(); + assertEquals(Type.STATE, collection2SateMessage.getType(), "State message is expected after all records in a stream are emitted"); + + assertFalse(collection2.hasNext()); + } + } diff --git a/docs/integrations/sources/mongodb-v2.md b/docs/integrations/sources/mongodb-v2.md index cbffe3f8094c..dfd5de71bb39 100644 --- a/docs/integrations/sources/mongodb-v2.md +++ b/docs/integrations/sources/mongodb-v2.md @@ -214,7 +214,8 @@ For more information regarding configuration parameters, please see [MongoDb Doc | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------| -| 1.2.4 | 2024-01-26 | [34573](https://github.com/airbytehq/airbyte/pull/34573) | Adopt CDK v0.16.0. | +| 1.2.5 | 2024-01-29 | [34573](https://github.com/airbytehq/airbyte/pull/34573) | Allow resuming an initial snapshot when Id type is not of default ObjectId . | +| 1.2.4 | 2024-01-26 | [34573](https://github.com/airbytehq/airbyte/pull/34573) | Adopt CDK v0.16.0. | | 1.2.3 | 2024-01-18 | [34364](https://github.com/airbytehq/airbyte/pull/34364) | Add additional logging for resume token + reduce discovery size to 10. | | 1.2.2 | 2024-01-16 | [34314](https://github.com/airbytehq/airbyte/pull/34314) | Reduce minimum document discovery size to 100. | | 1.2.1 | 2023-12-18 | [33549](https://github.com/airbytehq/airbyte/pull/33549) | Add logging to understand op log size. | From 1da8866d06897cd43d95dbb3ecf65fa4c8533a82 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 31 Jan 2024 10:40:30 +0100 Subject: [PATCH 36/96] Publish to pypi batch4 (#34666) --- .../connectors/source-facebook-pages/metadata.yaml | 4 ++++ .../connectors/source-fastbill/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-fauna/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-file/metadata.yaml | 4 ++++ .../source-firebase-realtime-database/metadata.yaml | 4 ++++ .../connectors/source-firebolt/metadata.yaml | 4 ++++ .../connectors/source-flexport/metadata.yaml | 4 ++++ .../connectors/source-freshcaller/metadata.yaml | 4 ++++ .../connectors/source-freshsales/metadata.yaml | 4 ++++ .../connectors/source-freshservice/metadata.yaml | 4 ++++ .../connectors/source-fullstory/metadata.yaml | 4 ++++ .../connectors/source-gainsight-px/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-gcs/metadata.yaml | 5 +++++ airbyte-integrations/connectors/source-genesys/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-getlago/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-github/metadata.yaml | 4 ++++ .../connectors/source-glassfrog/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-gnews/metadata.yaml | 5 +++++ .../connectors/source-gocardless/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-gong/metadata.yaml | 4 ++++ 20 files changed, 82 insertions(+) diff --git a/airbyte-integrations/connectors/source-facebook-pages/metadata.yaml b/airbyte-integrations/connectors/source-facebook-pages/metadata.yaml index d8dd267fcc57..1101c253d8ce 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/metadata.yaml +++ b/airbyte-integrations/connectors/source-facebook-pages/metadata.yaml @@ -15,6 +15,10 @@ data: icon: facebook.svg license: ELv2 name: Facebook Pages + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-facebook-pages registries: cloud: enabled: false # hide from cloud until https://github.com/airbytehq/airbyte/issues/25515 is finished diff --git a/airbyte-integrations/connectors/source-fastbill/metadata.yaml b/airbyte-integrations/connectors/source-fastbill/metadata.yaml index 9a6e795f561d..faf0afa33809 100644 --- a/airbyte-integrations/connectors/source-fastbill/metadata.yaml +++ b/airbyte-integrations/connectors/source-fastbill/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - "*" + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-fastbill registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-fauna/metadata.yaml b/airbyte-integrations/connectors/source-fauna/metadata.yaml index 602dccd287e4..cf36ff0c13ec 100644 --- a/airbyte-integrations/connectors/source-fauna/metadata.yaml +++ b/airbyte-integrations/connectors/source-fauna/metadata.yaml @@ -8,6 +8,10 @@ data: icon: fauna.svg license: MIT name: Fauna + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-fauna registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-file/metadata.yaml b/airbyte-integrations/connectors/source-file/metadata.yaml index a2c748e050d3..6d1fc12f20f1 100644 --- a/airbyte-integrations/connectors/source-file/metadata.yaml +++ b/airbyte-integrations/connectors/source-file/metadata.yaml @@ -17,6 +17,10 @@ data: icon: file.svg license: MIT name: File (CSV, JSON, Excel, Feather, Parquet) + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-file registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/metadata.yaml b/airbyte-integrations/connectors/source-firebase-realtime-database/metadata.yaml index abd8e66c3414..3d694aa9f2f3 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/metadata.yaml +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/metadata.yaml @@ -10,6 +10,10 @@ data: githubIssueLabel: source-firebase-realtime-database license: MIT name: Firebase Realtime Database + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-firebase-realtime-database registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-firebolt/metadata.yaml b/airbyte-integrations/connectors/source-firebolt/metadata.yaml index d56f04699b78..824d857b5d7e 100644 --- a/airbyte-integrations/connectors/source-firebolt/metadata.yaml +++ b/airbyte-integrations/connectors/source-firebolt/metadata.yaml @@ -8,6 +8,10 @@ data: icon: firebolt.svg license: MIT name: Firebolt + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-firebolt registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-flexport/metadata.yaml b/airbyte-integrations/connectors/source-flexport/metadata.yaml index deb99f7ca238..88b125bacb23 100644 --- a/airbyte-integrations/connectors/source-flexport/metadata.yaml +++ b/airbyte-integrations/connectors/source-flexport/metadata.yaml @@ -3,6 +3,10 @@ data: hosts: - api.flexport.com - flexport.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-flexport registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-freshcaller/metadata.yaml b/airbyte-integrations/connectors/source-freshcaller/metadata.yaml index ac2d4fb29261..15eaf3b06ea1 100644 --- a/airbyte-integrations/connectors/source-freshcaller/metadata.yaml +++ b/airbyte-integrations/connectors/source-freshcaller/metadata.yaml @@ -8,6 +8,10 @@ data: icon: freshcaller.svg license: MIT name: Freshcaller + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-freshcaller registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-freshsales/metadata.yaml b/airbyte-integrations/connectors/source-freshsales/metadata.yaml index afb486eb0a95..5abeea00cab7 100644 --- a/airbyte-integrations/connectors/source-freshsales/metadata.yaml +++ b/airbyte-integrations/connectors/source-freshsales/metadata.yaml @@ -20,6 +20,10 @@ data: icon: freshsales.svg license: MIT name: Freshsales + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-freshsales registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-freshservice/metadata.yaml b/airbyte-integrations/connectors/source-freshservice/metadata.yaml index 43c5dd48d13f..7c2055f9a0dd 100644 --- a/airbyte-integrations/connectors/source-freshservice/metadata.yaml +++ b/airbyte-integrations/connectors/source-freshservice/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - ${domain_name}/api/v2 + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-freshservice registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-fullstory/metadata.yaml b/airbyte-integrations/connectors/source-fullstory/metadata.yaml index a645935c33eb..c2083f3631c8 100644 --- a/airbyte-integrations/connectors/source-fullstory/metadata.yaml +++ b/airbyte-integrations/connectors/source-fullstory/metadata.yaml @@ -8,6 +8,10 @@ data: icon: fullstory.svg license: MIT name: Fullstory + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-fullstory registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-gainsight-px/metadata.yaml b/airbyte-integrations/connectors/source-gainsight-px/metadata.yaml index 2d340cc0adaf..0a65633f7dcc 100644 --- a/airbyte-integrations/connectors/source-gainsight-px/metadata.yaml +++ b/airbyte-integrations/connectors/source-gainsight-px/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.aptrinsic.com/v1 + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-gainsight-px registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-gcs/metadata.yaml b/airbyte-integrations/connectors/source-gcs/metadata.yaml index b303f6085b7a..d707202aa035 100644 --- a/airbyte-integrations/connectors/source-gcs/metadata.yaml +++ b/airbyte-integrations/connectors/source-gcs/metadata.yaml @@ -14,6 +14,11 @@ data: icon: gcs.svg license: ELv2 name: GCS + remoteRegistries: + pypi: + enabled: false + # TODO: Set enabled=true after `airbyte-lib-validate-source` is passing. + packageName: airbyte-source-gcs registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-genesys/metadata.yaml b/airbyte-integrations/connectors/source-genesys/metadata.yaml index f6522beea79e..0bad42c982d3 100644 --- a/airbyte-integrations/connectors/source-genesys/metadata.yaml +++ b/airbyte-integrations/connectors/source-genesys/metadata.yaml @@ -8,6 +8,10 @@ data: icon: genesys.svg license: MIT name: Genesys + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-genesys registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-getlago/metadata.yaml b/airbyte-integrations/connectors/source-getlago/metadata.yaml index 70779de3b016..de14460248c0 100644 --- a/airbyte-integrations/connectors/source-getlago/metadata.yaml +++ b/airbyte-integrations/connectors/source-getlago/metadata.yaml @@ -8,6 +8,10 @@ data: icon: getlago.svg license: MIT name: Lago + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-getlago registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-github/metadata.yaml b/airbyte-integrations/connectors/source-github/metadata.yaml index 91e3e7878d7d..0c9904bf27e8 100644 --- a/airbyte-integrations/connectors/source-github/metadata.yaml +++ b/airbyte-integrations/connectors/source-github/metadata.yaml @@ -18,6 +18,10 @@ data: license: MIT maxSecondsBetweenMessages: 5400 name: GitHub + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-github registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-glassfrog/metadata.yaml b/airbyte-integrations/connectors/source-glassfrog/metadata.yaml index b75ae80dbeb8..edf2c6a6fba6 100644 --- a/airbyte-integrations/connectors/source-glassfrog/metadata.yaml +++ b/airbyte-integrations/connectors/source-glassfrog/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.glassfrog.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-glassfrog registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-gnews/metadata.yaml b/airbyte-integrations/connectors/source-gnews/metadata.yaml index 0f8a7ba6b189..3c2eb68602d9 100644 --- a/airbyte-integrations/connectors/source-gnews/metadata.yaml +++ b/airbyte-integrations/connectors/source-gnews/metadata.yaml @@ -8,6 +8,11 @@ data: icon: gnews.svg license: MIT name: GNews + remoteRegistries: + pypi: + enabled: false + # TODO: Set enabled=true after `airbyte-lib-validate-source` is passing. + packageName: airbyte-source-gnews registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-gocardless/metadata.yaml b/airbyte-integrations/connectors/source-gocardless/metadata.yaml index d49af139e575..3d8b577d174a 100644 --- a/airbyte-integrations/connectors/source-gocardless/metadata.yaml +++ b/airbyte-integrations/connectors/source-gocardless/metadata.yaml @@ -8,6 +8,10 @@ data: icon: gocardless.svg license: MIT name: GoCardless + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-gocardless registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-gong/metadata.yaml b/airbyte-integrations/connectors/source-gong/metadata.yaml index 08864fcb3190..42fb73e25f10 100644 --- a/airbyte-integrations/connectors/source-gong/metadata.yaml +++ b/airbyte-integrations/connectors/source-gong/metadata.yaml @@ -8,6 +8,10 @@ data: icon: gong.svg license: MIT name: Gong + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-gong registries: cloud: enabled: false From b8a806d3f8e9b44aaf64692be5299d9ecbc05292 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 31 Jan 2024 12:08:50 +0100 Subject: [PATCH 37/96] airbyte-ci: Test pypi published properly (#34689) --- .github/actions/run-airbyte-ci/action.yml | 5 +--- airbyte-ci/connectors/pipelines/README.md | 4 +++- .../airbyte_ci/connectors/publish/commands.py | 13 +++++++++-- .../airbyte_ci/connectors/publish/context.py | 4 +++- .../airbyte_ci/connectors/publish/pipeline.py | 2 +- .../airbyte_ci/poetry/publish/commands.py | 3 ++- .../connectors/pipelines/pipelines/consts.py | 1 + .../pipelines/pipelines/helpers/pip.py | 20 ++++++---------- .../contexts/python_registry_publish.py | 3 +++ .../connectors/pipelines/pyproject.toml | 2 +- .../pipelines/tests/test_helpers/test_pip.py | 23 +++++++++++++++++++ .../tests/test_poetry/test_poetry_publish.py | 1 + 12 files changed, 57 insertions(+), 24 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/tests/test_helpers/test_pip.py diff --git a/.github/actions/run-airbyte-ci/action.yml b/.github/actions/run-airbyte-ci/action.yml index 753c041727e7..87d8b6c8f787 100644 --- a/.github/actions/run-airbyte-ci/action.yml +++ b/.github/actions/run-airbyte-ci/action.yml @@ -79,10 +79,6 @@ inputs: python_registry_token: description: "Python registry API token to publish python package" required: false - python_registry_url: - description: "Python registry URL to publish python package" - default: "https://upload.pypi.org/legacy/" - required: false runs: using: "composite" @@ -140,6 +136,7 @@ runs: PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} PYTHON_REGISTRY_TOKEN: ${{ inputs.python_registry_token }} PYTHON_REGISTRY_URL: ${{ inputs.python_registry_url }} + PYTHON_REGISTRY_CHECK_URL: ${{ inputs.python_registry_check_url }} S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ inputs.s3_build_cache_access_key_id }} S3_BUILD_CACHE_SECRET_KEY: ${{ inputs.s3_build_cache_secret_key }} SENTRY_DSN: ${{ inputs.sentry_dsn }} diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 3e8ccd074580..20490091b520 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -397,6 +397,7 @@ Publish all connectors modified in the head commit: `airbyte-ci connectors --mod | `--ci-requirements` | False | | | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. | | `--python-registry-token` | False | | `PYTHON_REGISTRY_TOKEN` | The API token to authenticate with the registry. For pypi, the `pypi-` prefix needs to be specified | | `--python-registry-url` | False | https://upload.pypi.org/legacy/ | `PYTHON_REGISTRY_URL` | The python registry to publish to. Defaults to main pypi | +| `--python-registry-check-url` | False | https://pypi.org/pypi | `PYTHON_REGISTRY_CHECK_URL` | The python registry url to check whether a package is published already | I've added an empty "Default" column, and you can fill in the default values as needed. @@ -421,7 +422,7 @@ flowchart TD If `remoteRegistries.pypi.enabled` in the connector metadata is set to `true`, the connector will be published to the python registry. To do so, the `--python-registry-token` and `--python-registry-url` options are used to authenticate with the registry and publish the connector. -If the current version of the connector is already published to the registry, the publish will be skipped. +If the current version of the connector is already published to the registry, the publish will be skipped (the `--python-registry-check-url` is used for the check). On a pre-release, the connector will be published as a `.dev` version. @@ -609,6 +610,7 @@ E.G.: running `pytest` on a specific test folder: | Version | PR | Description | | ------- | ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| 3.10.0 | [#34606](https://github.com/airbytehq/airbyte/pull/34606) | Allow configuration of separate check URL to check whether package exists already. | | 3.9.0 | [#34606](https://github.com/airbytehq/airbyte/pull/34606) | Allow configuration of python registry URL via environment variable. | | 3.8.1 | [#34607](https://github.com/airbytehq/airbyte/pull/34607) | Improve gradle dependency cache volume protection. | | 3.8.0 | [#34316](https://github.com/airbytehq/airbyte/pull/34316) | Expose Dagger engine image name in `--ci-requirements` and add `--ci-requirements` to the `airbyte-ci` root command group. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py index c5fbfc414486..fc34936e248b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py @@ -11,7 +11,7 @@ from pipelines.cli.click_decorators import click_ci_requirements_option from pipelines.cli.confirm_prompt import confirm from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand -from pipelines.consts import DEFAULT_PYTHON_PACKAGE_REGISTRY_URL, ContextState +from pipelines.consts import DEFAULT_PYTHON_PACKAGE_REGISTRY_CHECK_URL, DEFAULT_PYTHON_PACKAGE_REGISTRY_URL, ContextState from pipelines.helpers.utils import fail_if_missing_docker_hub_creds @@ -67,11 +67,18 @@ ) @click.option( "--python-registry-url", - help="Which python registry registry to publish to. If not set, the default pypi is used. For test pypi, use https://test.pypi.org/legacy/", + help="Which python registry url to publish to. If not set, the default pypi is used. For test pypi, use https://test.pypi.org/legacy/", type=click.STRING, default=DEFAULT_PYTHON_PACKAGE_REGISTRY_URL, envvar="PYTHON_REGISTRY_URL", ) +@click.option( + "--python-registry-check-url", + help="Which url to check whether a certain version is published already. If not set, the default pypi is used. For test pypi, use https://test.pypi.org/pypi/", + type=click.STRING, + default=DEFAULT_PYTHON_PACKAGE_REGISTRY_CHECK_URL, + envvar="PYTHON_REGISTRY_CHECK_URL", +) @click.pass_context async def publish( ctx: click.Context, @@ -84,6 +91,7 @@ async def publish( slack_channel: str, python_registry_token: str, python_registry_url: str, + python_registry_check_url: str, ) -> bool: ctx.obj["spec_cache_gcs_credentials"] = spec_cache_gcs_credentials ctx.obj["spec_cache_bucket_name"] = spec_cache_bucket_name @@ -126,6 +134,7 @@ async def publish( use_local_cdk=ctx.obj.get("use_local_cdk"), python_registry_token=python_registry_token, python_registry_url=python_registry_url, + python_registry_check_url=python_registry_check_url, ) for connector in ctx.obj["selected_connectors_with_modified_files"] ] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py index 55136809d8ce..a497629daec1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py @@ -37,6 +37,8 @@ def __init__( is_local: bool, git_branch: str, git_revision: str, + python_registry_url: str, + python_registry_check_url: str, gha_workflow_run_url: Optional[str] = None, dagger_logs_url: Optional[str] = None, pipeline_start_timestamp: Optional[int] = None, @@ -47,7 +49,6 @@ def __init__( s3_build_cache_secret_key: Optional[str] = None, use_local_cdk: bool = False, python_registry_token: Optional[str] = None, - python_registry_url: Optional[str] = None, ) -> None: self.pre_release = pre_release self.spec_cache_bucket_name = spec_cache_bucket_name @@ -56,6 +57,7 @@ def __init__( self.metadata_service_gcs_credentials = sanitize_gcs_credentials(metadata_service_gcs_credentials) self.python_registry_token = python_registry_token self.python_registry_url = python_registry_url + self.python_registry_check_url = python_registry_check_url pipeline_name = f"Publish {connector.technical_name}" pipeline_name = pipeline_name + " (pre-release)" if pre_release else pipeline_name diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index ac7514439cb0..35f1a961d09e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -60,7 +60,7 @@ class CheckPythonRegistryPackageDoesNotExist(Step): async def _run(self) -> StepResult: is_published = is_package_published( - self.context.package_metadata.name, self.context.package_metadata.version, self.context.registry + self.context.package_metadata.name, self.context.package_metadata.version, self.context.registry_check_url ) if is_published: return StepResult( diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py index 16bded0b0c34..0eac52947bf1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py @@ -14,7 +14,7 @@ from pipelines.airbyte_ci.steps.python_registry import PublishToPythonRegistry from pipelines.cli.confirm_prompt import confirm from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand -from pipelines.consts import DEFAULT_PYTHON_PACKAGE_REGISTRY_URL +from pipelines.consts import DEFAULT_PYTHON_PACKAGE_REGISTRY_CHECK_URL, DEFAULT_PYTHON_PACKAGE_REGISTRY_URL from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context from pipelines.models.contexts.python_registry_publish import PythonRegistryPublishContext from pipelines.models.steps import StepStatus @@ -87,6 +87,7 @@ async def publish( ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], python_registry_token=python_registry_token, registry=python_registry_url, + registry_check_url=DEFAULT_PYTHON_PACKAGE_REGISTRY_CHECK_URL, package_path=ctx.obj["package_path"], package_name=publish_name, version=publish_version, diff --git a/airbyte-ci/connectors/pipelines/pipelines/consts.py b/airbyte-ci/connectors/pipelines/pipelines/consts.py index 966c484b15f1..e9fe8ee5d1a5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/consts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/consts.py @@ -61,6 +61,7 @@ STORAGE_DRIVER = "fuse-overlayfs" SETUP_PY_FILE_PATH = "setup.py" DEFAULT_PYTHON_PACKAGE_REGISTRY_URL = "https://upload.pypi.org/legacy/" +DEFAULT_PYTHON_PACKAGE_REGISTRY_CHECK_URL = "https://pypi.org/pypi" class CIContext(str, Enum): diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/pip.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/pip.py index e8b1d0ed0dad..5076e6b401ce 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/pip.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/pip.py @@ -1,27 +1,21 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. from typing import Optional -from urllib.parse import urlparse import requests # type: ignore def is_package_published(package_name: Optional[str], version: Optional[str], registry_url: str) -> bool: """ - Check if a package with a specific version is published on PyPI or Test PyPI. - - :param package_name: The name of the package to check. - :param version: The version of the package. - :param test_pypi: Set to True to check on Test PyPI, False for regular PyPI. - :return: True if the package is found with the specified version, False otherwise. + Check if a package with a specific version is published on a python registry. """ if not package_name or not version: return False - parsed_registry_url = urlparse(registry_url) - base_url = f"{parsed_registry_url.scheme}://{parsed_registry_url.netloc}" - - url = f"{base_url}/{package_name}/{version}/json" + url = f"{registry_url}/{package_name}/{version}/json" - response = requests.get(url) - return response.status_code == 200 + try: + response = requests.get(url) + return response.status_code == 200 + except requests.exceptions.ConnectionError: + return False diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/python_registry_publish.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/python_registry_publish.py index a189b476b892..2b406e0d7887 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/python_registry_publish.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/python_registry_publish.py @@ -21,6 +21,7 @@ class PythonRegistryPublishContext(PipelineContext): def __init__( self, python_registry_token: str, + registry_check_url: str, package_path: str, report_output_prefix: str, is_local: bool, @@ -38,6 +39,7 @@ def __init__( ) -> None: self.python_registry_token = python_registry_token self.registry = registry + self.registry_check_url = registry_check_url self.package_path = package_path self.package_metadata = PythonPackageMetadata(package_name, version) @@ -87,6 +89,7 @@ async def from_publish_connector_context( pypi_context = cls( python_registry_token=str(connector_context.python_registry_token), registry=str(connector_context.python_registry_url), + registry_check_url=str(connector_context.python_registry_check_url), package_path=str(connector_context.connector.code_directory), package_name=current_metadata["remoteRegistries"]["pypi"]["packageName"], version=version, diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 8828208d99dd..5471d7d63c45 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "3.9.0" +version = "3.10.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_pip.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_pip.py new file mode 100644 index 000000000000..26605c675849 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_pip.py @@ -0,0 +1,23 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import pytest +from pipelines.helpers.pip import is_package_published + + +@pytest.mark.parametrize( + "package_name, version, registry_url, expected", + [ + pytest.param(None, None, "https://pypi.org/pypi", False, id="package_name and version are None"), + pytest.param(None, "0.2.0", "https://pypi.org/pypi", False, id="package_name is None"), + pytest.param("airbyte-source-pokeapi", None, "https://pypi.org/pypi", False, id="version is None"), + pytest.param("airbyte-source-pokeapi", "0.2.0", "https://pypi.org/pypi", True, id="published on pypi"), + pytest.param("airbyte-source-pokeapi", "0.1.0", "https://pypi.org/pypi", False, id="version not published on pypi"), + pytest.param("airbyte-source-nonexisting", "0.1.0", "https://pypi.org/pypi", False, id="package not published on pypi"), + pytest.param("airbyte-source-pokeapi", "0.2.1", "https://test.pypi.org/pypi", True, id="published on test.pypi"), + pytest.param("airbyte-source-pokeapi", "0.1.0", "https://test.pypi.org/pypi", False, id="version not published on test.pypi"), + pytest.param("airbyte-source-nonexisting", "0.1.0", "https://test.pypi.org/pypi", False, id="package not published on test.pypi"), + pytest.param("airbyte-source-pokeapi", "0.2.0", "https://some-non-existing-host.com", False, id="host does not exist"), + ], +) +def test_is_package_published(package_name, version, registry_url, expected): + assert is_package_published(package_name, version, registry_url) == expected diff --git a/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py b/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py index ee345b418380..69fb4699c989 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py +++ b/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py @@ -24,6 +24,7 @@ def context(dagger_client: Client): version="0.2.0", python_registry_token="test", package_name="test", + registry_check_url="http://local_registry:8080/", registry="http://local_registry:8080/", is_local=True, git_branch="test", From b8e4871e30440514dd27dc19259b041063e1a202 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 31 Jan 2024 12:10:46 +0100 Subject: [PATCH 38/96] Publish to pypi batch5 (#34668) --- .../connectors/source-google-analytics-v4/metadata.yaml | 5 +++++ .../connectors/source-google-directory/metadata.yaml | 4 ++++ .../source-google-pagespeed-insights/metadata.yaml | 4 ++++ .../connectors/source-google-search-console/metadata.yaml | 5 +++++ .../connectors/source-google-webfonts/metadata.yaml | 4 ++++ .../source-google-workspace-admin-reports/metadata.yaml | 5 +++++ .../connectors/source-greenhouse/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-gridly/metadata.yaml | 4 ++++ .../connectors/source-gutendex/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-harness/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-harvest/metadata.yaml | 4 ++++ .../connectors/source-hellobaton/metadata.yaml | 4 ++++ .../connectors/source-hubplanner/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-hubspot/metadata.yaml | 4 ++++ .../connectors/source-insightly/metadata.yaml | 4 ++++ .../connectors/source-instatus/metadata.yaml | 4 ++++ .../connectors/source-intercom/metadata.yaml | 5 +++++ .../connectors/source-intruder/metadata.yaml | 4 ++++ .../connectors/source-ip2whois/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-jira/metadata.yaml | 4 ++++ 20 files changed, 84 insertions(+) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml b/airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml index 0d3f2c4740fd..9307c73d16b6 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml @@ -20,6 +20,11 @@ data: icon: google-analytics.svg license: Elv2 name: Google Analytics (Universal Analytics) + remoteRegistries: + pypi: + enabled: false + # TODO: Set enabled=true after `airbyte-lib-validate-source` is passing. + packageName: airbyte-source-google-analytics-v4 registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-google-directory/metadata.yaml b/airbyte-integrations/connectors/source-google-directory/metadata.yaml index 9db4b72e67e8..eedf71a553af 100644 --- a/airbyte-integrations/connectors/source-google-directory/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-directory/metadata.yaml @@ -8,6 +8,10 @@ data: icon: googledirectory.svg license: MIT name: Google Directory + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-google-directory registries: cloud: dockerImageTag: 0.2.1 diff --git a/airbyte-integrations/connectors/source-google-pagespeed-insights/metadata.yaml b/airbyte-integrations/connectors/source-google-pagespeed-insights/metadata.yaml index 68f96b0db8e7..2bfe25c5eeae 100644 --- a/airbyte-integrations/connectors/source-google-pagespeed-insights/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-pagespeed-insights/metadata.yaml @@ -8,6 +8,10 @@ data: icon: google-pagespeed-insights.svg license: MIT name: Google PageSpeed Insights + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-google-pagespeed-insights registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-google-search-console/metadata.yaml b/airbyte-integrations/connectors/source-google-search-console/metadata.yaml index e3d321483fd5..b64ccc2913d5 100644 --- a/airbyte-integrations/connectors/source-google-search-console/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-search-console/metadata.yaml @@ -17,6 +17,11 @@ data: icon: googlesearchconsole.svg license: Elv2 name: Google Search Console + remoteRegistries: + pypi: + enabled: false + # TODO: Set enabled=true after `airbyte-lib-validate-source` is passing. + packageName: airbyte-source-google-search-console registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-google-webfonts/metadata.yaml b/airbyte-integrations/connectors/source-google-webfonts/metadata.yaml index 58d9d4bd572f..b60374d751b9 100644 --- a/airbyte-integrations/connectors/source-google-webfonts/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-webfonts/metadata.yaml @@ -8,6 +8,10 @@ data: icon: googleworkpace.svg license: MIT name: Google Webfonts + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-google-webfonts registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/metadata.yaml b/airbyte-integrations/connectors/source-google-workspace-admin-reports/metadata.yaml index a0b5a4ca2007..bfc8f0f6421e 100644 --- a/airbyte-integrations/connectors/source-google-workspace-admin-reports/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-workspace-admin-reports/metadata.yaml @@ -8,6 +8,11 @@ data: icon: googleworkpace.svg license: MIT name: Google Workspace Admin Reports + remoteRegistries: + pypi: + enabled: false + # TODO: Set enabled=true after `airbyte-lib-validate-source` is passing. + packageName: airbyte-source-google-workspace-admin-reports registries: cloud: dockerImageTag: 0.1.4 diff --git a/airbyte-integrations/connectors/source-greenhouse/metadata.yaml b/airbyte-integrations/connectors/source-greenhouse/metadata.yaml index e732c0f18fa6..98cbce12ec27 100644 --- a/airbyte-integrations/connectors/source-greenhouse/metadata.yaml +++ b/airbyte-integrations/connectors/source-greenhouse/metadata.yaml @@ -17,6 +17,10 @@ data: icon: greenhouse.svg license: MIT name: Greenhouse + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-greenhouse registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-gridly/metadata.yaml b/airbyte-integrations/connectors/source-gridly/metadata.yaml index 1c6747986c8c..11a5a90d789f 100644 --- a/airbyte-integrations/connectors/source-gridly/metadata.yaml +++ b/airbyte-integrations/connectors/source-gridly/metadata.yaml @@ -8,6 +8,10 @@ data: icon: gridly.svg license: MIT name: Gridly + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-gridly registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-gutendex/metadata.yaml b/airbyte-integrations/connectors/source-gutendex/metadata.yaml index 01e6a71bbaed..6624a7e221bd 100644 --- a/airbyte-integrations/connectors/source-gutendex/metadata.yaml +++ b/airbyte-integrations/connectors/source-gutendex/metadata.yaml @@ -7,6 +7,10 @@ data: githubIssueLabel: source-gutendex license: MIT name: Gutendex + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-gutendex registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-harness/metadata.yaml b/airbyte-integrations/connectors/source-harness/metadata.yaml index a33d83b93859..59b15a6a1b61 100644 --- a/airbyte-integrations/connectors/source-harness/metadata.yaml +++ b/airbyte-integrations/connectors/source-harness/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.harness.io + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-harness registries: oss: enabled: false diff --git a/airbyte-integrations/connectors/source-harvest/metadata.yaml b/airbyte-integrations/connectors/source-harvest/metadata.yaml index 0bb7f453aea5..a1c6ee65ed77 100644 --- a/airbyte-integrations/connectors/source-harvest/metadata.yaml +++ b/airbyte-integrations/connectors/source-harvest/metadata.yaml @@ -17,6 +17,10 @@ data: icon: harvest.svg license: MIT name: Harvest + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-harvest registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-hellobaton/metadata.yaml b/airbyte-integrations/connectors/source-hellobaton/metadata.yaml index f01eb8e06ffe..183bf69ea8ad 100644 --- a/airbyte-integrations/connectors/source-hellobaton/metadata.yaml +++ b/airbyte-integrations/connectors/source-hellobaton/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - ${company}.hellobaton.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-hellobaton registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-hubplanner/metadata.yaml b/airbyte-integrations/connectors/source-hubplanner/metadata.yaml index 59487dacb6f1..b14c0ba4e0b1 100644 --- a/airbyte-integrations/connectors/source-hubplanner/metadata.yaml +++ b/airbyte-integrations/connectors/source-hubplanner/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - "*" # Please change to the hostname of the source. + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-hubplanner registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-hubspot/metadata.yaml b/airbyte-integrations/connectors/source-hubspot/metadata.yaml index e68f2ffd9280..0a6414cf98bd 100644 --- a/airbyte-integrations/connectors/source-hubspot/metadata.yaml +++ b/airbyte-integrations/connectors/source-hubspot/metadata.yaml @@ -17,6 +17,10 @@ data: icon: hubspot.svg license: ELv2 name: HubSpot + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-hubspot registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-insightly/metadata.yaml b/airbyte-integrations/connectors/source-insightly/metadata.yaml index 78f6fc859bf6..7fb79460cecd 100644 --- a/airbyte-integrations/connectors/source-insightly/metadata.yaml +++ b/airbyte-integrations/connectors/source-insightly/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - TODO # Please change to the hostname of the source. + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-insightly registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-instatus/metadata.yaml b/airbyte-integrations/connectors/source-instatus/metadata.yaml index 67980e6c3fb3..225e47a7158b 100644 --- a/airbyte-integrations/connectors/source-instatus/metadata.yaml +++ b/airbyte-integrations/connectors/source-instatus/metadata.yaml @@ -8,6 +8,10 @@ data: icon: instatus.svg license: MIT name: Instatus + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-instatus registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-intercom/metadata.yaml b/airbyte-integrations/connectors/source-intercom/metadata.yaml index beb243445d31..8f86ceba16b3 100644 --- a/airbyte-integrations/connectors/source-intercom/metadata.yaml +++ b/airbyte-integrations/connectors/source-intercom/metadata.yaml @@ -17,6 +17,11 @@ data: icon: intercom.svg license: MIT name: Intercom + remoteRegistries: + pypi: + enabled: false + # TODO: Set enabled=true after `airbyte-lib-validate-source` is passing. + packageName: airbyte-source-intercom registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-intruder/metadata.yaml b/airbyte-integrations/connectors/source-intruder/metadata.yaml index a70265c693ff..49d692755d42 100644 --- a/airbyte-integrations/connectors/source-intruder/metadata.yaml +++ b/airbyte-integrations/connectors/source-intruder/metadata.yaml @@ -8,6 +8,10 @@ data: icon: intruder.svg license: MIT name: Intruder + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-intruder registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-ip2whois/metadata.yaml b/airbyte-integrations/connectors/source-ip2whois/metadata.yaml index a14ed278b363..03866ecbd8ca 100644 --- a/airbyte-integrations/connectors/source-ip2whois/metadata.yaml +++ b/airbyte-integrations/connectors/source-ip2whois/metadata.yaml @@ -8,6 +8,10 @@ data: icon: ip2whois.svg license: MIT name: IP2Whois + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-ip2whois registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-jira/metadata.yaml b/airbyte-integrations/connectors/source-jira/metadata.yaml index f10331aafc5d..51eb24b0430a 100644 --- a/airbyte-integrations/connectors/source-jira/metadata.yaml +++ b/airbyte-integrations/connectors/source-jira/metadata.yaml @@ -18,6 +18,10 @@ data: license: MIT maxSecondsBetweenMessages: 21600 name: Jira + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-jira registries: cloud: enabled: true From c881fe08006688f3beec882b4053cd8a808633a4 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 31 Jan 2024 13:17:53 +0100 Subject: [PATCH 39/96] Publish to pypi batch6 (#34672) --- .../connectors/source-k6-cloud/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-klarna/metadata.yaml | 4 ++++ .../connectors/source-klaus-api/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-klaviyo/metadata.yaml | 4 ++++ .../connectors/source-kustomer-singer/metadata.yaml | 5 +++++ airbyte-integrations/connectors/source-kyriba/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-kyve/metadata.yaml | 4 ++++ .../connectors/source-launchdarkly/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-lemlist/metadata.yaml | 4 ++++ .../connectors/source-lever-hiring/metadata.yaml | 4 ++++ .../connectors/source-linkedin-pages/metadata.yaml | 4 ++++ .../connectors/source-linnworks/metadata.yaml | 5 +++++ .../connectors/source-lokalise/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-looker/metadata.yaml | 4 ++++ .../connectors/source-mailerlite/metadata.yaml | 4 ++++ .../connectors/source-mailersend/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-mailgun/metadata.yaml | 4 ++++ .../connectors/source-mailjet-mail/metadata.yaml | 4 ++++ .../connectors/source-mailjet-sms/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-merge/metadata.yaml | 4 ++++ 20 files changed, 82 insertions(+) diff --git a/airbyte-integrations/connectors/source-k6-cloud/metadata.yaml b/airbyte-integrations/connectors/source-k6-cloud/metadata.yaml index f62b94eacbf6..f0c7664e3897 100644 --- a/airbyte-integrations/connectors/source-k6-cloud/metadata.yaml +++ b/airbyte-integrations/connectors/source-k6-cloud/metadata.yaml @@ -8,6 +8,10 @@ data: icon: k6cloud.svg license: MIT name: K6 Cloud + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-k6-cloud registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-klarna/metadata.yaml b/airbyte-integrations/connectors/source-klarna/metadata.yaml index 8eff9ee1fc7b..caf42f707603 100644 --- a/airbyte-integrations/connectors/source-klarna/metadata.yaml +++ b/airbyte-integrations/connectors/source-klarna/metadata.yaml @@ -5,6 +5,10 @@ data: - api.playground.klarna.com - api-${config.region}.klarna.com - api-${config.region}.playground.klarna.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-klarna registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-klaus-api/metadata.yaml b/airbyte-integrations/connectors/source-klaus-api/metadata.yaml index 24e538bf888a..2f4dbc01d8cf 100644 --- a/airbyte-integrations/connectors/source-klaus-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-klaus-api/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - "*" # Please change to the hostname of the source. + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-klaus-api registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-klaviyo/metadata.yaml b/airbyte-integrations/connectors/source-klaviyo/metadata.yaml index 71bcca6adeb0..1844fa9c85d9 100644 --- a/airbyte-integrations/connectors/source-klaviyo/metadata.yaml +++ b/airbyte-integrations/connectors/source-klaviyo/metadata.yaml @@ -14,6 +14,10 @@ data: icon: klaviyo.svg license: MIT name: Klaviyo + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-klaviyo registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-kustomer-singer/metadata.yaml b/airbyte-integrations/connectors/source-kustomer-singer/metadata.yaml index ede0b9624645..99cf6995426b 100644 --- a/airbyte-integrations/connectors/source-kustomer-singer/metadata.yaml +++ b/airbyte-integrations/connectors/source-kustomer-singer/metadata.yaml @@ -8,6 +8,11 @@ data: icon: kustomer.svg license: MIT name: Kustomer + remoteRegistries: + pypi: + enabled: false + # TODO: Set enabled=true after `airbyte-lib-validate-source` is passing. + packageName: airbyte-source-kustomer-singer registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-kyriba/metadata.yaml b/airbyte-integrations/connectors/source-kyriba/metadata.yaml index 347d7b22f2fa..bed9c52b8527 100644 --- a/airbyte-integrations/connectors/source-kyriba/metadata.yaml +++ b/airbyte-integrations/connectors/source-kyriba/metadata.yaml @@ -8,6 +8,10 @@ data: icon: kyriba.svg license: MIT name: Kyriba + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-kyriba registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-kyve/metadata.yaml b/airbyte-integrations/connectors/source-kyve/metadata.yaml index 6f2b6cfc27ec..c95fa5e33341 100644 --- a/airbyte-integrations/connectors/source-kyve/metadata.yaml +++ b/airbyte-integrations/connectors/source-kyve/metadata.yaml @@ -9,6 +9,10 @@ data: icon: icon.svg license: MIT name: KYVE + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-kyve registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-launchdarkly/metadata.yaml b/airbyte-integrations/connectors/source-launchdarkly/metadata.yaml index d55516006941..c93a438bd591 100644 --- a/airbyte-integrations/connectors/source-launchdarkly/metadata.yaml +++ b/airbyte-integrations/connectors/source-launchdarkly/metadata.yaml @@ -8,6 +8,10 @@ data: icon: launchdarkly.svg license: MIT name: LaunchDarkly + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-launchdarkly registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-lemlist/metadata.yaml b/airbyte-integrations/connectors/source-lemlist/metadata.yaml index cf4bd7c33ff8..6fbfb68fb760 100644 --- a/airbyte-integrations/connectors/source-lemlist/metadata.yaml +++ b/airbyte-integrations/connectors/source-lemlist/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.lemlist.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-lemlist registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-lever-hiring/metadata.yaml b/airbyte-integrations/connectors/source-lever-hiring/metadata.yaml index 9a49336207cb..93c0e331a63b 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/metadata.yaml +++ b/airbyte-integrations/connectors/source-lever-hiring/metadata.yaml @@ -8,6 +8,10 @@ data: icon: leverhiring.svg license: MIT name: Lever Hiring + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-lever-hiring registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-linkedin-pages/metadata.yaml b/airbyte-integrations/connectors/source-linkedin-pages/metadata.yaml index f2c1338be532..f5d1196e7b6c 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/metadata.yaml +++ b/airbyte-integrations/connectors/source-linkedin-pages/metadata.yaml @@ -12,6 +12,10 @@ data: icon: linkedin.svg license: MIT name: LinkedIn Pages + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-linkedin-pages registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-linnworks/metadata.yaml b/airbyte-integrations/connectors/source-linnworks/metadata.yaml index 326dbedaa270..a40652bfd7ff 100644 --- a/airbyte-integrations/connectors/source-linnworks/metadata.yaml +++ b/airbyte-integrations/connectors/source-linnworks/metadata.yaml @@ -8,6 +8,11 @@ data: icon: linnworks.svg license: MIT name: Linnworks + remoteRegistries: + pypi: + enabled: false + # TODO: Set enabled=true after `airbyte-lib-validate-source` is passing. + packageName: airbyte-source-linnworks registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-lokalise/metadata.yaml b/airbyte-integrations/connectors/source-lokalise/metadata.yaml index 079f05bcb4f6..2f222da3c397 100644 --- a/airbyte-integrations/connectors/source-lokalise/metadata.yaml +++ b/airbyte-integrations/connectors/source-lokalise/metadata.yaml @@ -8,6 +8,10 @@ data: icon: lokalise.svg license: MIT name: Lokalise + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-lokalise registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-looker/metadata.yaml b/airbyte-integrations/connectors/source-looker/metadata.yaml index 10e3a4cfe079..3d2415c0cd29 100644 --- a/airbyte-integrations/connectors/source-looker/metadata.yaml +++ b/airbyte-integrations/connectors/source-looker/metadata.yaml @@ -8,6 +8,10 @@ data: icon: looker.svg license: MIT name: Looker + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-looker registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-mailerlite/metadata.yaml b/airbyte-integrations/connectors/source-mailerlite/metadata.yaml index fae3ac67fb61..b89c174e8d0f 100644 --- a/airbyte-integrations/connectors/source-mailerlite/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailerlite/metadata.yaml @@ -8,6 +8,10 @@ data: icon: mailerlite.svg license: MIT name: MailerLite + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-mailerlite registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-mailersend/metadata.yaml b/airbyte-integrations/connectors/source-mailersend/metadata.yaml index 436e6a8514ed..97799066552c 100644 --- a/airbyte-integrations/connectors/source-mailersend/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailersend/metadata.yaml @@ -8,6 +8,10 @@ data: icon: mailersend.svg license: MIT name: MailerSend + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-mailersend registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-mailgun/metadata.yaml b/airbyte-integrations/connectors/source-mailgun/metadata.yaml index 7d3e7d30ee15..669d11d1ef41 100644 --- a/airbyte-integrations/connectors/source-mailgun/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailgun/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - https://api.mailgun.net/ + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-mailgun registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-mailjet-mail/metadata.yaml b/airbyte-integrations/connectors/source-mailjet-mail/metadata.yaml index 3a0ae99fd35a..ce0bd1f6e3b2 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailjet-mail/metadata.yaml @@ -8,6 +8,10 @@ data: icon: mailjetmail.svg license: MIT name: Mailjet Mail + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-mailjet-mail registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-mailjet-sms/metadata.yaml b/airbyte-integrations/connectors/source-mailjet-sms/metadata.yaml index a68dbed823e2..307381bb7f6c 100644 --- a/airbyte-integrations/connectors/source-mailjet-sms/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailjet-sms/metadata.yaml @@ -8,6 +8,10 @@ data: icon: mailjetsms.svg license: MIT name: Mailjet SMS + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-mailjet-sms registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-merge/metadata.yaml b/airbyte-integrations/connectors/source-merge/metadata.yaml index 13c6df4feb03..8a311f0a516d 100644 --- a/airbyte-integrations/connectors/source-merge/metadata.yaml +++ b/airbyte-integrations/connectors/source-merge/metadata.yaml @@ -8,6 +8,10 @@ data: icon: merge.svg license: MIT name: Merge + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-merge registries: cloud: enabled: false From 410fc8a840047a9b3fc2835037cfcf16da1bb2ff Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 31 Jan 2024 14:14:15 +0100 Subject: [PATCH 40/96] Publish to pypi batch7 (#34673) --- .../connectors/source-metabase/metadata.yaml | 4 ++++ .../connectors/source-microsoft-dataverse/metadata.yaml | 4 ++++ .../connectors/source-microsoft-onedrive/metadata.yaml | 5 +++++ .../connectors/source-microsoft-teams/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-monday/metadata.yaml | 5 +++++ .../connectors/source-my-hours/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-n8n/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-nasa/metadata.yaml | 4 ++++ .../connectors/source-netsuite/metadata.yaml | 4 ++++ .../connectors/source-news-api/metadata.yaml | 4 ++++ .../connectors/source-newsdata/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-notion/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-nytimes/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-okta/metadata.yaml | 4 ++++ .../connectors/source-omnisend/metadata.yaml | 4 ++++ .../connectors/source-onesignal/metadata.yaml | 4 ++++ .../connectors/source-open-exchange-rates/metadata.yaml | 4 ++++ .../connectors/source-openweather/metadata.yaml | 4 ++++ .../connectors/source-opsgenie/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-orb/metadata.yaml | 4 ++++ 20 files changed, 82 insertions(+) diff --git a/airbyte-integrations/connectors/source-metabase/metadata.yaml b/airbyte-integrations/connectors/source-metabase/metadata.yaml index c6289bf09e4f..2ee8fe2744b6 100644 --- a/airbyte-integrations/connectors/source-metabase/metadata.yaml +++ b/airbyte-integrations/connectors/source-metabase/metadata.yaml @@ -15,6 +15,10 @@ data: icon: metabase.svg license: MIT name: Metabase + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-metabase registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/metadata.yaml b/airbyte-integrations/connectors/source-microsoft-dataverse/metadata.yaml index b91c74f4767e..30fbf7e9da33 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/metadata.yaml +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/metadata.yaml @@ -8,6 +8,10 @@ data: icon: microsoftdataverse.svg license: MIT name: Microsoft Dataverse + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-microsoft-dataverse registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml b/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml index b6ef12226f71..c9f69260cd8d 100644 --- a/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml @@ -6,6 +6,11 @@ data: hosts: - graph.microsoft.com - login.microsoftonline.com + remoteRegistries: + pypi: + enabled: false + # TODO: Set enabled=true after `airbyte-lib-validate-source` is passing. + packageName: airbyte-source-microsoft-onedrive registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-microsoft-teams/metadata.yaml b/airbyte-integrations/connectors/source-microsoft-teams/metadata.yaml index a554cf83e32e..f9307d6d500c 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/metadata.yaml +++ b/airbyte-integrations/connectors/source-microsoft-teams/metadata.yaml @@ -8,6 +8,10 @@ data: icon: microsoft-teams.svg license: MIT name: Microsoft teams + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-microsoft-teams registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-monday/metadata.yaml b/airbyte-integrations/connectors/source-monday/metadata.yaml index d5a3a9f0f41c..e30b06c3cd2e 100644 --- a/airbyte-integrations/connectors/source-monday/metadata.yaml +++ b/airbyte-integrations/connectors/source-monday/metadata.yaml @@ -34,6 +34,11 @@ data: icon: monday.svg license: MIT name: Monday + remoteRegistries: + pypi: + enabled: false + # TODO: Set enabled=true after `airbyte-lib-validate-source` is passing. + packageName: airbyte-source-monday registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-my-hours/metadata.yaml b/airbyte-integrations/connectors/source-my-hours/metadata.yaml index ab7bf1e3e5be..27da449d698a 100644 --- a/airbyte-integrations/connectors/source-my-hours/metadata.yaml +++ b/airbyte-integrations/connectors/source-my-hours/metadata.yaml @@ -8,6 +8,10 @@ data: icon: my-hours.svg license: MIT name: My Hours + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-my-hours registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-n8n/metadata.yaml b/airbyte-integrations/connectors/source-n8n/metadata.yaml index f34328d9ee75..afabe0ee24aa 100644 --- a/airbyte-integrations/connectors/source-n8n/metadata.yaml +++ b/airbyte-integrations/connectors/source-n8n/metadata.yaml @@ -8,6 +8,10 @@ data: icon: n8n.svg license: MIT name: n8n + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-n8n registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-nasa/metadata.yaml b/airbyte-integrations/connectors/source-nasa/metadata.yaml index 1cd017174495..8bc0a54a45a0 100644 --- a/airbyte-integrations/connectors/source-nasa/metadata.yaml +++ b/airbyte-integrations/connectors/source-nasa/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.nasa.gov + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-nasa registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-netsuite/metadata.yaml b/airbyte-integrations/connectors/source-netsuite/metadata.yaml index c2451ba1bfa3..2ce3fb426c0c 100644 --- a/airbyte-integrations/connectors/source-netsuite/metadata.yaml +++ b/airbyte-integrations/connectors/source-netsuite/metadata.yaml @@ -12,6 +12,10 @@ data: icon: netsuite.svg license: MIT name: Netsuite + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-netsuite registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-news-api/metadata.yaml b/airbyte-integrations/connectors/source-news-api/metadata.yaml index d33ea74b3d18..e4f2c6d7e45b 100644 --- a/airbyte-integrations/connectors/source-news-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-news-api/metadata.yaml @@ -8,6 +8,10 @@ data: icon: newsapi.svg license: MIT name: News API + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-news-api registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-newsdata/metadata.yaml b/airbyte-integrations/connectors/source-newsdata/metadata.yaml index e27f67670526..1ae1a6bb448c 100644 --- a/airbyte-integrations/connectors/source-newsdata/metadata.yaml +++ b/airbyte-integrations/connectors/source-newsdata/metadata.yaml @@ -7,6 +7,10 @@ data: githubIssueLabel: source-newsdata license: MIT name: Newsdata + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-newsdata registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-notion/metadata.yaml b/airbyte-integrations/connectors/source-notion/metadata.yaml index 69f953d8a5e2..97d50e3ba0a6 100644 --- a/airbyte-integrations/connectors/source-notion/metadata.yaml +++ b/airbyte-integrations/connectors/source-notion/metadata.yaml @@ -17,6 +17,10 @@ data: icon: notion.svg license: MIT name: Notion + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-notion registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-nytimes/metadata.yaml b/airbyte-integrations/connectors/source-nytimes/metadata.yaml index fa67687b7293..abb5e9e358cc 100644 --- a/airbyte-integrations/connectors/source-nytimes/metadata.yaml +++ b/airbyte-integrations/connectors/source-nytimes/metadata.yaml @@ -8,6 +8,10 @@ data: icon: nytimes.svg license: MIT name: New York Times + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-nytimes registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-okta/metadata.yaml b/airbyte-integrations/connectors/source-okta/metadata.yaml index 011fb07a3b8b..2a7b9541606e 100644 --- a/airbyte-integrations/connectors/source-okta/metadata.yaml +++ b/airbyte-integrations/connectors/source-okta/metadata.yaml @@ -12,6 +12,10 @@ data: icon: okta.svg license: MIT name: Okta + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-okta registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-omnisend/metadata.yaml b/airbyte-integrations/connectors/source-omnisend/metadata.yaml index 1f350fea734d..3b27557209fa 100644 --- a/airbyte-integrations/connectors/source-omnisend/metadata.yaml +++ b/airbyte-integrations/connectors/source-omnisend/metadata.yaml @@ -8,6 +8,10 @@ data: icon: omnisend.svg license: MIT name: Omnisend + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-omnisend registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-onesignal/metadata.yaml b/airbyte-integrations/connectors/source-onesignal/metadata.yaml index cedb38e32be9..1c57a5eebb55 100644 --- a/airbyte-integrations/connectors/source-onesignal/metadata.yaml +++ b/airbyte-integrations/connectors/source-onesignal/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - "onesignal.com" + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-onesignal registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/metadata.yaml b/airbyte-integrations/connectors/source-open-exchange-rates/metadata.yaml index 7c5f21637fae..4cce2373e1b4 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/metadata.yaml +++ b/airbyte-integrations/connectors/source-open-exchange-rates/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - openexchangerates.org + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-open-exchange-rates registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-openweather/metadata.yaml b/airbyte-integrations/connectors/source-openweather/metadata.yaml index b15d5744b4f9..e92518c4f92b 100644 --- a/airbyte-integrations/connectors/source-openweather/metadata.yaml +++ b/airbyte-integrations/connectors/source-openweather/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.openweathermap.org + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-openweather registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-opsgenie/metadata.yaml b/airbyte-integrations/connectors/source-opsgenie/metadata.yaml index 17edbf7b36e5..995487473494 100644 --- a/airbyte-integrations/connectors/source-opsgenie/metadata.yaml +++ b/airbyte-integrations/connectors/source-opsgenie/metadata.yaml @@ -7,6 +7,10 @@ data: githubIssueLabel: source-opsgenie license: MIT name: Opsgenie + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-opsgenie registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-orb/metadata.yaml b/airbyte-integrations/connectors/source-orb/metadata.yaml index 16fbdbb5ab0a..1c5c21ce8624 100644 --- a/airbyte-integrations/connectors/source-orb/metadata.yaml +++ b/airbyte-integrations/connectors/source-orb/metadata.yaml @@ -8,6 +8,10 @@ data: icon: orb.svg license: MIT name: Orb + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-orb registries: cloud: enabled: true From 0c2d92e6392335d69497d227e87d1fb7e2d6cc71 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Wed, 31 Jan 2024 10:27:40 -0300 Subject: [PATCH 41/96] Kubernetes docs: update instructions to use external database (#34604) --- .../on-kubernetes-via-helm.md | 70 ++++++++++++++++--- 1 file changed, 61 insertions(+), 9 deletions(-) diff --git a/docs/deploying-airbyte/on-kubernetes-via-helm.md b/docs/deploying-airbyte/on-kubernetes-via-helm.md index 060e5a6e6654..162a2de4b9cf 100644 --- a/docs/deploying-airbyte/on-kubernetes-via-helm.md +++ b/docs/deploying-airbyte/on-kubernetes-via-helm.md @@ -187,20 +187,72 @@ After updating `values.yaml` simply upgrade your chart by running command: helm upgrade -f path/to/values.yaml %release_name% airbyte/airbyte ``` -### Database external secrets +### External Airbyte Database -If you're using external DB secrets, then provide them in `values.yaml` under global.database section in the following format: +::info +This was tested using [Airbyte Helm Chart Version 0.50.13](https://artifacthub.io/packages/helm/airbyte/airbyte/0.50.13). +Previous or newer version can change how the external database can be configured. +::: -```text +The Airbyte Database only works with Postgres 13. +Make sure the database is accessible inside the cluster using `busy-box` service using `telnet` or `ping` command. + +:::warning +If you're using the external database for the first time you must ensure the database you're going to use exists. The default database Airbyte will try to use is `airbyte` but you can modified it in the `values.yaml`. +::: + +:::warning +You can use only one database to a one Airbyte Helm deployment. If you try to use the same database for a different deployment it will have conflict with Temporal internal databases. +::: + +Create a Kubernetes secret to store the database password. +Save the file as `db-secrets.yaml`. +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: db-secrets +type: Opaque +stringData: + DATABASE_PASSWORD: +``` + +Run `kubectl apply -f db-secrets.yaml -n ` to create the secret in the namespace you're using Airbyte. + +Afterward, modify the following blocks in the Helm Chart `values.yaml` file: +```yaml +postgresql: + # Change the value from true to false. + enabled: false +``` +Then: +```yaml +externalDatabase: + # Add the host, username and database name you're using. + host: + user: + database: + password: "" + existingSecret: "db-secrets" + existingSecretPasswordKey: "DATABASE_PASSWORD" + port: 5432 + jdbcUrl: "" +``` +Keep password empty as the Chart will use the `db-secrets` value. +Edit only the host, username, and database name. If your database is using a differnet `port` or need an special `jdbcUrl` you can edit here. +This wasn't fully tested yet. + +Next, reference the secret in the global section: +```yaml +global: database: - secretName: "myOctaviaSecret" - secretValue: "postgresql-password" - host: "example.com" - port: "5432" + secretName: "db-secrets" + secretValue: "DATABASE_PASSWORD" ``` -And upgrade the chart by running: +Unfortunately, the `airbyte-bootloader` configuration uses this variable. Future improvements are planned. +Upgrade the chart by running: ```shell -helm upgrade -f path/to/values.yaml %release_name% airbyte/airbyte +helm upgrade --install %RELEASE_NAME% airbyte/airbyte -n --values /path/to/values.yaml --version 0.50.13 ``` From 045ce953164f44f39d0cf4656b861305720163dd Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Wed, 31 Jan 2024 08:48:38 -0500 Subject: [PATCH 42/96] Update file-based connectors for compatibility with concurrent CDK (#34681) --- .../source-azure-blob-storage/metadata.yaml | 2 +- .../source-azure-blob-storage/setup.py | 2 +- .../source_azure_blob_storage/run.py | 10 +++++- .../source_azure_blob_storage/source.py | 7 ++-- .../connectors/source-gcs/metadata.yaml | 2 +- .../connectors/source-gcs/setup.py | 2 +- .../connectors/source-gcs/source_gcs/run.py | 34 +++++++++++++++++-- .../source-gcs/source_gcs/source.py | 7 ++-- .../source-google-drive/metadata.yaml | 2 +- .../connectors/source-google-drive/setup.py | 2 +- .../source_google_drive/run.py | 8 ++++- .../source_google_drive/source.py | 11 +++--- .../source-microsoft-onedrive/metadata.yaml | 2 +- .../source-microsoft-onedrive/setup.py | 2 +- .../source_microsoft_onedrive/run.py | 8 ++++- .../source_microsoft_onedrive/source.py | 11 +++--- .../sources/azure-blob-storage.md | 1 + docs/integrations/sources/gcs.md | 21 ++++++------ docs/integrations/sources/google-drive.md | 1 + .../sources/microsoft-onedrive.md | 1 + 20 files changed, 98 insertions(+), 38 deletions(-) diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml b/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml index eca1b2e4c108..9abbaec55465 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml +++ b/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: file connectorType: source definitionId: fdaaba68-4875-4ed9-8fcd-4ae1e0a25093 - dockerImageTag: 0.3.2 + dockerImageTag: 0.3.3 dockerRepository: airbyte/source-azure-blob-storage documentationUrl: https://docs.airbyte.com/integrations/sources/azure-blob-storage githubIssueLabel: source-azure-blob-storage diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/setup.py b/airbyte-integrations/connectors/source-azure-blob-storage/setup.py index 5260ac12c058..827a6fb87060 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/setup.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[file-based]==0.59.2", # pinned until compatible with https://github.com/airbytehq/airbyte/pull/34411 + "airbyte-cdk[file-based]>=0.60.1", "smart_open[azure]", "pytz", ] diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/run.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/run.py index 404d919b60f3..a671d836526a 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/run.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/run.py @@ -14,8 +14,16 @@ def run(): args = sys.argv[1:] catalog_path = AirbyteEntrypoint.extract_catalog(args) + config_path = AirbyteEntrypoint.extract_config(args) + state_path = AirbyteEntrypoint.extract_state(args) try: - source = SourceAzureBlobStorage(SourceAzureBlobStorageStreamReader(), Config, catalog_path) + source = SourceAzureBlobStorage( + SourceAzureBlobStorageStreamReader(), + Config, + SourceAzureBlobStorage.read_catalog(catalog_path) if catalog_path else None, + SourceAzureBlobStorage.read_config(config_path) if catalog_path else None, + SourceAzureBlobStorage.read_state(state_path) if catalog_path else None, + ) except Exception: print( AirbyteMessage( diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/source.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/source.py index 419119bb3ef8..792dcdfe2221 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/source.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/source.py @@ -11,14 +11,15 @@ class SourceAzureBlobStorage(FileBasedSource): - def read_config(self, config_path: str) -> Mapping[str, Any]: + @classmethod + def read_config(cls, config_path: str) -> Mapping[str, Any]: """ Used to override the default read_config so that when the new file-based Azure Blob Storage connector processes a config in the legacy format, it can be transformed into the new config. This happens in entrypoint before we validate the config against the new spec. """ - config = super().read_config(config_path) - if not self._is_v1_config(config): + config = FileBasedSource.read_config(config_path) + if not cls._is_v1_config(config): converted_config = LegacyConfigTransformer.convert(config) emit_configuration_as_airbyte_control_message(converted_config) return converted_config diff --git a/airbyte-integrations/connectors/source-gcs/metadata.yaml b/airbyte-integrations/connectors/source-gcs/metadata.yaml index d707202aa035..6ec717a2dfe6 100644 --- a/airbyte-integrations/connectors/source-gcs/metadata.yaml +++ b/airbyte-integrations/connectors/source-gcs/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: file connectorType: source definitionId: 2a8c41ae-8c23-4be0-a73f-2ab10ca1a820 - dockerImageTag: 0.3.5 + dockerImageTag: 0.3.6 dockerRepository: airbyte/source-gcs documentationUrl: https://docs.airbyte.com/integrations/sources/gcs githubIssueLabel: source-gcs diff --git a/airbyte-integrations/connectors/source-gcs/setup.py b/airbyte-integrations/connectors/source-gcs/setup.py index 830498574d7a..1e174a438b84 100644 --- a/airbyte-integrations/connectors/source-gcs/setup.py +++ b/airbyte-integrations/connectors/source-gcs/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[file-based]==0.59.2", # pinned until compatible with https://github.com/airbytehq/airbyte/pull/34411 + "airbyte-cdk[file-based]>=0.60.1", "google-cloud-storage==2.12.0", "smart-open[s3]==5.1.0", "pandas==1.5.3", diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/run.py b/airbyte-integrations/connectors/source-gcs/source_gcs/run.py index d91a6b40df2a..c4536b2d14e8 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/run.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/run.py @@ -4,13 +4,41 @@ import sys +import traceback +from datetime import datetime from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch +from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type from source_gcs import Config, Cursor, SourceGCS, SourceGCSStreamReader def run(): _args = sys.argv[1:] - catalog_path = AirbyteEntrypoint.extract_catalog(_args) - source = SourceGCS(SourceGCSStreamReader(), Config, catalog_path, cursor_cls=Cursor) - launch(source, sys.argv[1:]) + try: + catalog_path = AirbyteEntrypoint.extract_catalog(_args) + config_path = AirbyteEntrypoint.extract_config(_args) + state_path = AirbyteEntrypoint.extract_state(_args) + source = SourceGCS( + SourceGCSStreamReader(), + Config, + SourceGCS.read_catalog(catalog_path) if catalog_path else None, + SourceGCS.read_config(config_path) if config_path else None, + SourceGCS.read_state(state_path) if state_path else None, + cursor_cls=Cursor, + ) + except Exception: + print( + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.ERROR, + emitted_at=int(datetime.now().timestamp() * 1000), + error=AirbyteErrorTraceMessage( + message="Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance.", + stack_trace=traceback.format_exc(), + ), + ), + ).json() + ) + else: + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/source.py b/airbyte-integrations/connectors/source-gcs/source_gcs/source.py index b33ff2b87eb4..7152d9ec5e11 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/source.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/source.py @@ -12,13 +12,14 @@ class SourceGCS(FileBasedSource): - def read_config(self, config_path: str) -> Mapping[str, Any]: + @classmethod + def read_config(cls, config_path: str) -> Mapping[str, Any]: """ Override the default read_config to transform the legacy config format into the new one before validating it against the new spec. """ - config = super().read_config(config_path) - if not self._is_file_based_config(config): + config = FileBasedSource.read_config(config_path) + if not cls._is_file_based_config(config): parsed_legacy_config = SourceGCSSpec(**config) converted_config = LegacyConfigTransformer.convert(parsed_legacy_config) emit_configuration_as_airbyte_control_message(converted_config) diff --git a/airbyte-integrations/connectors/source-google-drive/metadata.yaml b/airbyte-integrations/connectors/source-google-drive/metadata.yaml index 6f4839d112c9..0a37d17fa989 100644 --- a/airbyte-integrations/connectors/source-google-drive/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-drive/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: file connectorType: source definitionId: 9f8dda77-1048-4368-815b-269bf54ee9b8 - dockerImageTag: 0.0.7 + dockerImageTag: 0.0.8 dockerRepository: airbyte/source-google-drive githubIssueLabel: source-google-drive icon: google-drive.svg diff --git a/airbyte-integrations/connectors/source-google-drive/setup.py b/airbyte-integrations/connectors/source-google-drive/setup.py index 858c7019384a..0685947ad335 100644 --- a/airbyte-integrations/connectors/source-google-drive/setup.py +++ b/airbyte-integrations/connectors/source-google-drive/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[file-based]==0.59.2", # pinned until compatible with https://github.com/airbytehq/airbyte/pull/34411 + "airbyte-cdk[file-based]>=0.60.1", "google-api-python-client==2.104.0", "google-auth-httplib2==0.1.1", "google-auth-oauthlib==1.1.0", diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/run.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/run.py index a5605f39c2b1..d9d56d62740b 100644 --- a/airbyte-integrations/connectors/source-google-drive/source_google_drive/run.py +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/run.py @@ -13,5 +13,11 @@ def run(): args = sys.argv[1:] catalog_path = AirbyteEntrypoint.extract_catalog(args) - source = SourceGoogleDrive(catalog_path) + config_path = AirbyteEntrypoint.extract_config(args) + state_path = AirbyteEntrypoint.extract_state(args) + source = SourceGoogleDrive( + SourceGoogleDrive.read_catalog(catalog_path) if catalog_path else None, + SourceGoogleDrive.read_config(config_path) if config_path else None, + SourceGoogleDrive.read_state(state_path) if state_path else None, + ) launch(source, args) diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/source.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/source.py index fe49fba7fe8c..479711be83ee 100644 --- a/airbyte-integrations/connectors/source-google-drive/source_google_drive/source.py +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/source.py @@ -1,21 +1,24 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Any +from typing import Any, Mapping, Optional -from airbyte_cdk.models import AdvancedAuth, ConnectorSpecification, OAuthConfigSpecification +from airbyte_cdk.models import AdvancedAuth, ConfiguredAirbyteCatalog, ConnectorSpecification, OAuthConfigSpecification from airbyte_cdk.sources.file_based.file_based_source import FileBasedSource from airbyte_cdk.sources.file_based.stream.cursor.default_file_based_cursor import DefaultFileBasedCursor +from airbyte_cdk.sources.source import TState from source_google_drive.spec import SourceGoogleDriveSpec from source_google_drive.stream_reader import SourceGoogleDriveStreamReader class SourceGoogleDrive(FileBasedSource): - def __init__(self, catalog_path: str): + def __init__(self, catalog: Optional[ConfiguredAirbyteCatalog], config: Optional[Mapping[str, Any]], state: Optional[TState]): super().__init__( stream_reader=SourceGoogleDriveStreamReader(), spec_class=SourceGoogleDriveSpec, - catalog_path=catalog_path, + catalog=catalog, + config=config, + state=state, cursor_cls=DefaultFileBasedCursor, ) diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml b/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml index c9f69260cd8d..fb4d15c8e029 100644 --- a/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml @@ -21,7 +21,7 @@ data: connectorSubtype: api connectorType: source definitionId: 01d1c685-fd4a-4837-8f4c-93fe5a0d2188 - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.5 dockerRepository: airbyte/source-microsoft-onedrive githubIssueLabel: source-microsoft-onedrive icon: microsoft-onedrive.svg diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/setup.py b/airbyte-integrations/connectors/source-microsoft-onedrive/setup.py index bc93b53a4004..342c12983500 100644 --- a/airbyte-integrations/connectors/source-microsoft-onedrive/setup.py +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[file-based]==0.59.2", # pinned until compatible with https://github.com/airbytehq/airbyte/pull/34411 + "airbyte-cdk[file-based]>=0.60.1", "msal~=1.25.0", "Office365-REST-Python-Client~=2.5.2", "smart-open~=6.4.0", diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/run.py b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/run.py index 2141089780b9..f083b8c70abc 100644 --- a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/run.py +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/run.py @@ -13,5 +13,11 @@ def run(): args = sys.argv[1:] catalog_path = AirbyteEntrypoint.extract_catalog(args) - source = SourceMicrosoftOneDrive(catalog_path) + config_path = AirbyteEntrypoint.extract_config(args) + state_path = AirbyteEntrypoint.extract_state(args) + source = SourceMicrosoftOneDrive( + SourceMicrosoftOneDrive.read_catalog(catalog_path) if catalog_path else None, + SourceMicrosoftOneDrive.read_config(config_path) if config_path else None, + SourceMicrosoftOneDrive.read_state(state_path) if state_path else None, + ) launch(source, args) diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/source.py b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/source.py index 87cf25732fd8..f317a0990e84 100644 --- a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/source.py +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/source.py @@ -1,21 +1,24 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Any +from typing import Any, Mapping, Optional -from airbyte_cdk.models import AdvancedAuth, ConnectorSpecification, OAuthConfigSpecification +from airbyte_cdk.models import AdvancedAuth, ConfiguredAirbyteCatalog, ConnectorSpecification, OAuthConfigSpecification from airbyte_cdk.sources.file_based.file_based_source import FileBasedSource from airbyte_cdk.sources.file_based.stream.cursor.default_file_based_cursor import DefaultFileBasedCursor +from airbyte_cdk.sources.source import TState from source_microsoft_onedrive.spec import SourceMicrosoftOneDriveSpec from source_microsoft_onedrive.stream_reader import SourceMicrosoftOneDriveStreamReader class SourceMicrosoftOneDrive(FileBasedSource): - def __init__(self, catalog_path: str): + def __init__(self, catalog: Optional[ConfiguredAirbyteCatalog], config: Optional[Mapping[str, Any]], state: Optional[TState]): super().__init__( stream_reader=SourceMicrosoftOneDriveStreamReader(), spec_class=SourceMicrosoftOneDriveSpec, - catalog_path=catalog_path, + catalog=catalog, + config=config, + state=state, cursor_cls=DefaultFileBasedCursor, ) diff --git a/docs/integrations/sources/azure-blob-storage.md b/docs/integrations/sources/azure-blob-storage.md index 01d082c71ca0..b41ccb856f58 100644 --- a/docs/integrations/sources/azure-blob-storage.md +++ b/docs/integrations/sources/azure-blob-storage.md @@ -193,6 +193,7 @@ To perform the text extraction from PDF and Docx files, the connector uses the [ | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------| +| 0.3.3 | 2024-01-30 | [34681](https://github.com/airbytehq/airbyte/pull/34681) | Unpin CDK version to make compatible with the Concurrent CDK | | 0.3.2 | 2024-01-30 | [34661](https://github.com/airbytehq/airbyte/pull/34661) | Pin CDK version until upgrade for compatibility with the Concurrent CDK | | 0.3.1 | 2024-01-10 | [34084](https://github.com/airbytehq/airbyte/pull/34084) | Fix bug for running check with document file format | | 0.3.0 | 2023-12-14 | [33411](https://github.com/airbytehq/airbyte/pull/33411) | Bump CDK version to auto-set primary key for document file streams and support raw txt files | diff --git a/docs/integrations/sources/gcs.md b/docs/integrations/sources/gcs.md index 58e24a17dcc4..85e429d98f36 100644 --- a/docs/integrations/sources/gcs.md +++ b/docs/integrations/sources/gcs.md @@ -146,13 +146,14 @@ Leaving this field blank (default option) will disallow escaping. ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------| -| 0.3.5 | 2024-01-30 | [34661](https://github.com/airbytehq/airbyte/pull/34661) | Pin CDK version until upgrade for compatibility with the Concurrent CDK | -| 0.3.4 | 2024-01-11 | [34158](https://github.com/airbytehq/airbyte/pull/34158) | Fix issue in stream reader for document file type parser | -| 0.3.3 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | -| 0.3.2 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Improve document file type parser | -| 0.3.1 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | -| 0.3.0 | 2023-10-11 | [31212](https://github.com/airbytehq/airbyte/pull/31212) | Migrated to file based CDK | -| 0.2.0 | 2023-06-26 | [27725](https://github.com/airbytehq/airbyte/pull/27725) | License Update: Elv2 | -| 0.1.0 | 2023-02-16 | [23186](https://github.com/airbytehq/airbyte/pull/23186) | New Source: GCS | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------| +| 0.3.6 | 2024-01-30 | [34681](https://github.com/airbytehq/airbyte/pull/34681) | Unpin CDK version to make compatible with the Concurrent CDK | +| 0.3.5 | 2024-01-30 | [34661](https://github.com/airbytehq/airbyte/pull/34661) | Pin CDK version until upgrade for compatibility with the Concurrent CDK | +| 0.3.4 | 2024-01-11 | [34158](https://github.com/airbytehq/airbyte/pull/34158) | Fix issue in stream reader for document file type parser | +| 0.3.3 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | +| 0.3.2 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Improve document file type parser | +| 0.3.1 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | +| 0.3.0 | 2023-10-11 | [31212](https://github.com/airbytehq/airbyte/pull/31212) | Migrated to file based CDK | +| 0.2.0 | 2023-06-26 | [27725](https://github.com/airbytehq/airbyte/pull/27725) | License Update: Elv2 | +| 0.1.0 | 2023-02-16 | [23186](https://github.com/airbytehq/airbyte/pull/23186) | New Source: GCS | diff --git a/docs/integrations/sources/google-drive.md b/docs/integrations/sources/google-drive.md index ca98dfb3129e..b7ea26e0e32e 100644 --- a/docs/integrations/sources/google-drive.md +++ b/docs/integrations/sources/google-drive.md @@ -247,6 +247,7 @@ Before parsing each document, the connector exports Google Document files to Doc | Version | Date | Pull Request | Subject | |---------|------------|-----------------------------------------------------------|--------------------------------------------------------------| +| 0.0.8 | 2024-01-30 | [34681](https://github.com/airbytehq/airbyte/pull/34681) | Unpin CDK version to make compatible with the Concurrent CDK | | 0.0.7 | 2024-01-30 | [34661](https://github.com/airbytehq/airbyte/pull/34661) | Pin CDK version until upgrade for compatibility with the Concurrent CDK | | 0.0.6 | 2023-12-16 | [33414](https://github.com/airbytehq/airbyte/pull/33414) | Prepare for airbyte-lib | | 0.0.5 | 2023-12-14 | [33411](https://github.com/airbytehq/airbyte/pull/33411) | Bump CDK version to auto-set primary key for document file streams and support raw txt files | diff --git a/docs/integrations/sources/microsoft-onedrive.md b/docs/integrations/sources/microsoft-onedrive.md index 4697bce4734c..a298e5b98db5 100644 --- a/docs/integrations/sources/microsoft-onedrive.md +++ b/docs/integrations/sources/microsoft-onedrive.md @@ -121,6 +121,7 @@ The connector is restricted by normal Microsoft Graph [requests limitation](http | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------| +| 0.1.5 | 2024-01-30 | [34681](https://github.com/airbytehq/airbyte/pull/34681) | Unpin CDK version to make compatible with the Concurrent CDK | | 0.1.4 | 2024-01-30 | [34661](https://github.com/airbytehq/airbyte/pull/34661) | Pin CDK version until upgrade for compatibility with the Concurrent CDK | | 0.1.3 | 2024-01-24 | [34478](https://github.com/airbytehq/airbyte/pull/34478) | Fix OAuth | | 0.1.2 | 2021-12-22 | [33745](https://github.com/airbytehq/airbyte/pull/33745) | Add ql and sl to metadata | From 65a7f4237fe04f107dedfc319ae87193b52aa19d Mon Sep 17 00:00:00 2001 From: Anton Karpets Date: Wed, 31 Jan 2024 16:33:04 +0200 Subject: [PATCH 43/96] =?UTF-8?q?=F0=9F=9A=A8=F0=9F=9A=A8=F0=9F=90=9BSourc?= =?UTF-8?q?e=20Hubspot:=20update=20marketing=5Femails=20stream=20schema=20?= =?UTF-8?q?(#34492)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../integration_tests/expected_records.jsonl | 62 ++-- .../connectors/source-hubspot/metadata.yaml | 10 +- .../source_hubspot/schemas/contact_lists.json | 6 + .../source_hubspot/schemas/engagements.json | 18 ++ .../source_hubspot/schemas/forms.json | 17 ++ .../schemas/marketing_emails.json | 286 +++++++++--------- .../source_hubspot/schemas/workflows.json | 6 + .../source-hubspot/unit_tests/test_source.py | 8 +- .../sources/hubspot-migrations.md | 40 +++ docs/integrations/sources/hubspot.md | 1 + 10 files changed, 272 insertions(+), 182 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl index 0109a703be27..a0f564927da3 100644 --- a/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl @@ -1,22 +1,22 @@ {"stream": "campaigns", "data": {"id": 243851494, "lastUpdatedTime": 1675121674226, "appId": 113, "appName": "Batch", "contentId": 100523515217, "subject": "test", "name": "test", "counters": {"dropped": 1}, "lastProcessingFinishedAt": 1675121674000, "lastProcessingStartedAt": 1675121671000, "lastProcessingStateChangeAt": 1675121674000, "numIncluded": 1, "processingState": "DONE", "type": "BATCH_EMAIL", "counters_dropped": 1}, "emitted_at": 1697714185530} {"stream": "campaigns", "data": {"id": 115429485, "lastUpdatedTime": 1615506409286, "appId": 113, "appName": "Batch", "contentId": 42931043849, "subject": "Test subj", "name": "Test subj", "counters": {"processed": 1, "deferred": 1, "mta_dropped": 1, "dropped": 3, "sent": 0}, "lastProcessingFinishedAt": 1615504712000, "lastProcessingStartedAt": 1615504687000, "lastProcessingStateChangeAt": 1615504712000, "numIncluded": 3, "processingState": "DONE", "type": "BATCH_EMAIL", "counters_processed": 1, "counters_deferred": 1, "counters_mta_dropped": 1, "counters_dropped": 3, "counters_sent": 0}, "emitted_at": 1697714185763} -{"stream": "companies", "data": {"id": "4992593519", "properties": {"about_us": null, "address": null, "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": null, "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-10T07:58:09.554000+00:00", "custom_company_property": null, "days_to_close": null, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "airbyte.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": null, "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": "2021-05-21T10:17:06.028000+00:00", "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": null, "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_num_page_views": null, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": null, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": null, "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": "2021-05-21T10:17:28.964000+00:00", "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-01-26T11:45:49.817000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": null, "hs_num_blockers": null, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": null, "hs_num_decision_makers": null, "hs_num_open_deals": 1, "hs_object_id": 4992593519, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": null, "hs_predictivecontactscore_v2": null, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.5476861596107483, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": 76121938222, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2020-12-10T07:58:09.554000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": "opportunity", "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Airbyte test1", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_associated_deals": 1, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 200, "phone": "+1 415-307-4864", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": null, "recent_deal_close_date": null, "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": null, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;segment;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "website": "airbyte.io", "zip": "94114"}, "createdAt": "2020-12-10T07:58:09.554Z", "updatedAt": "2023-01-26T11:45:49.817Z", "archived": false, "properties_about_us": null, "properties_address": null, "properties_address2": null, "properties_annualrevenue": null, "properties_city": "San Francisco", "properties_closedate": null, "properties_closedate_timestamp_earliest_value_a2a17e6e": null, "properties_country": "United States", "properties_createdate": "2020-12-10T07:58:09.554000+00:00", "properties_custom_company_property": null, "properties_days_to_close": null, "properties_description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_domain": "airbyte.io", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_facebook_company_page": null, "properties_facebookfans": null, "properties_first_contact_createdate": null, "properties_first_contact_createdate_timestamp_earliest_value_78b50eea": null, "properties_first_conversion_date": null, "properties_first_conversion_date_timestamp_earliest_value_61f58f2c": null, "properties_first_conversion_event_name": null, "properties_first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "properties_first_deal_created_date": "2021-05-21T10:17:06.028000+00:00", "properties_founded_year": "2020", "properties_googleplus_page": null, "properties_hs_additional_domains": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_first_timestamp": null, "properties_hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "properties_hs_analytics_latest_source": null, "properties_hs_analytics_latest_source_data_1": null, "properties_hs_analytics_latest_source_data_2": null, "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_num_page_views": null, "properties_hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "properties_hs_analytics_num_visits": null, "properties_hs_analytics_num_visits_cardinality_sum_53d952a6": null, "properties_hs_analytics_source": null, "properties_hs_analytics_source_data_1": null, "properties_hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "properties_hs_analytics_source_data_2": null, "properties_hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "properties_hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "properties_hs_annual_revenue_currency_code": "USD", "properties_hs_avatar_filemanager_key": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": "2021-05-21T10:17:28.964000+00:00", "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": null, "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_ideal_customer_profile": null, "properties_hs_is_target_account": null, "properties_hs_last_booked_meeting_date": null, "properties_hs_last_logged_call_date": null, "properties_hs_last_open_task_date": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": "2023-01-26T11:45:49.817000+00:00", "properties_hs_latest_createdate_of_active_subscriptions": null, "properties_hs_latest_meeting_activity": null, "properties_hs_lead_status": null, "properties_hs_merged_object_ids": null, "properties_hs_num_blockers": null, "properties_hs_num_child_companies": 0, "properties_hs_num_contacts_with_buying_roles": null, "properties_hs_num_decision_makers": null, "properties_hs_num_open_deals": 1, "properties_hs_object_id": 4992593519, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_parent_company_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": null, "properties_hs_predictivecontactscore_v2": null, "properties_hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_target_account": null, "properties_hs_target_account_probability": 0.5476861596107483, "properties_hs_target_account_recommendation_snooze_time": null, "properties_hs_target_account_recommendation_state": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": 76121938222, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": null, "properties_hs_total_deal_value": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2020-12-10T07:58:09.554000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_is_public": false, "properties_lifecyclestage": "opportunity", "properties_linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "properties_linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_name": "Airbyte test1", "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_associated_deals": 1, "properties_num_contacted_notes": null, "properties_num_conversion_events": null, "properties_num_conversion_events_cardinality_sum_d095f14b": null, "properties_num_notes": null, "properties_numberofemployees": 200, "properties_phone": "+1 415-307-4864", "properties_recent_conversion_date": null, "properties_recent_conversion_date_timestamp_latest_value_72856da1": null, "properties_recent_conversion_event_name": null, "properties_recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_state": "CA", "properties_timezone": "America/Los_Angeles", "properties_total_money_raised": null, "properties_total_revenue": null, "properties_twitterbio": null, "properties_twitterfollowers": null, "properties_twitterhandle": "AirbyteHQ", "properties_type": null, "properties_web_technologies": "slack;segment;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "properties_website": "airbyte.io", "properties_zip": "94114"}, "emitted_at": 1697714187356} -{"stream": "companies", "data": {"id": "5000526215", "properties": {"about_us": null, "address": null, "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": "2023-04-04T15:00:58.081000+00:00", "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-11T01:27:40.002000+00:00", "custom_company_property": null, "days_to_close": 844, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "dataline.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": "2020-12-11T01:29:50.116000+00:00", "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": "2021-01-13T10:30:42.221000+00:00", "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": "OFFLINE", "hs_analytics_latest_source_data_1": "CONTACTS", "hs_analytics_latest_source_data_2": "CRM_UI", "hs_analytics_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "hs_analytics_num_page_views": 0, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": 0, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": "CRM_UI", "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": "2023-04-04T15:00:58.081000+00:00", "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": "2021-02-23T20:21:06.027000+00:00", "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": "2023-04-04T15:00:58.081000+00:00", "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-09-07T03:58:14.126000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": "5183403213", "hs_num_blockers": 0, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": 0, "hs_num_decision_makers": 0, "hs_num_open_deals": 2, "hs_object_id": 5000526215, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "companies-lifecycle-pipeline", "hs_predictivecontactscore_v2": 0.3, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.46257445216178894, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": 17093729103, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": 66508792054, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": 60010, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2020-12-11T01:27:40.002000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": "customer", "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Dataline", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 1, "num_associated_deals": 3, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 25, "phone": "", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": 60000, "recent_deal_close_date": "2023-04-04T14:59:45.103000+00:00", "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": 60000, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;segment;google_tag_manager;cloud_flare;google_analytics;intercom;lever;google_apps", "website": "dataline.io", "zip": ""}, "createdAt": "2020-12-11T01:27:40.002Z", "updatedAt": "2023-09-07T03:58:14.126Z", "archived": false, "contacts": ["151", "151"], "properties_about_us": null, "properties_address": null, "properties_address2": null, "properties_annualrevenue": null, "properties_city": "San Francisco", "properties_closedate": "2023-04-04T15:00:58.081000+00:00", "properties_closedate_timestamp_earliest_value_a2a17e6e": null, "properties_country": "United States", "properties_createdate": "2020-12-11T01:27:40.002000+00:00", "properties_custom_company_property": null, "properties_days_to_close": 844, "properties_description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_domain": "dataline.io", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_facebook_company_page": null, "properties_facebookfans": null, "properties_first_contact_createdate": "2020-12-11T01:29:50.116000+00:00", "properties_first_contact_createdate_timestamp_earliest_value_78b50eea": null, "properties_first_conversion_date": null, "properties_first_conversion_date_timestamp_earliest_value_61f58f2c": null, "properties_first_conversion_event_name": null, "properties_first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "properties_first_deal_created_date": "2021-01-13T10:30:42.221000+00:00", "properties_founded_year": "2020", "properties_googleplus_page": null, "properties_hs_additional_domains": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "properties_hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "properties_hs_analytics_latest_source": "OFFLINE", "properties_hs_analytics_latest_source_data_1": "CONTACTS", "properties_hs_analytics_latest_source_data_2": "CRM_UI", "properties_hs_analytics_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "properties_hs_analytics_num_page_views": 0, "properties_hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "properties_hs_analytics_num_visits": 0, "properties_hs_analytics_num_visits_cardinality_sum_53d952a6": null, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "CONTACTS", "properties_hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "properties_hs_analytics_source_data_2": "CRM_UI", "properties_hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "properties_hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "properties_hs_annual_revenue_currency_code": "USD", "properties_hs_avatar_filemanager_key": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_date_entered_customer": "2023-04-04T15:00:58.081000+00:00", "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": "2021-02-23T20:21:06.027000+00:00", "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": null, "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": "2023-04-04T15:00:58.081000+00:00", "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_ideal_customer_profile": null, "properties_hs_is_target_account": null, "properties_hs_last_booked_meeting_date": null, "properties_hs_last_logged_call_date": null, "properties_hs_last_open_task_date": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": "2023-09-07T03:58:14.126000+00:00", "properties_hs_latest_createdate_of_active_subscriptions": null, "properties_hs_latest_meeting_activity": null, "properties_hs_lead_status": null, "properties_hs_merged_object_ids": "5183403213", "properties_hs_num_blockers": 0, "properties_hs_num_child_companies": 0, "properties_hs_num_contacts_with_buying_roles": 0, "properties_hs_num_decision_makers": 0, "properties_hs_num_open_deals": 2, "properties_hs_object_id": 5000526215, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_parent_company_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "companies-lifecycle-pipeline", "properties_hs_predictivecontactscore_v2": 0.3, "properties_hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_target_account": null, "properties_hs_target_account_probability": 0.46257445216178894, "properties_hs_target_account_recommendation_snooze_time": null, "properties_hs_target_account_recommendation_state": null, "properties_hs_time_in_customer": 17093729103, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": 66508792054, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": null, "properties_hs_total_deal_value": 60010, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2020-12-11T01:27:40.002000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_is_public": false, "properties_lifecyclestage": "customer", "properties_linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "properties_linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_name": "Dataline", "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 1, "properties_num_associated_deals": 3, "properties_num_contacted_notes": null, "properties_num_conversion_events": null, "properties_num_conversion_events_cardinality_sum_d095f14b": null, "properties_num_notes": null, "properties_numberofemployees": 25, "properties_phone": "", "properties_recent_conversion_date": null, "properties_recent_conversion_date_timestamp_latest_value_72856da1": null, "properties_recent_conversion_event_name": null, "properties_recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "properties_recent_deal_amount": 60000, "properties_recent_deal_close_date": "2023-04-04T14:59:45.103000+00:00", "properties_state": "CA", "properties_timezone": "America/Los_Angeles", "properties_total_money_raised": null, "properties_total_revenue": 60000, "properties_twitterbio": null, "properties_twitterfollowers": null, "properties_twitterhandle": "AirbyteHQ", "properties_type": null, "properties_web_technologies": "slack;segment;google_tag_manager;cloud_flare;google_analytics;intercom;lever;google_apps", "properties_website": "dataline.io", "properties_zip": ""}, "emitted_at": 1697714187359} -{"stream": "companies", "data": {"id": "5000787595", "properties": {"about_us": null, "address": "2261 Market Street", "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": null, "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-11T01:28:27.673000+00:00", "custom_company_property": null, "days_to_close": null, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "Daxtarity.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": null, "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": null, "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": null, "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": "", "hs_analytics_latest_source_data_1": "", "hs_analytics_latest_source_data_2": "", "hs_analytics_latest_source_timestamp": null, "hs_analytics_num_page_views": null, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": null, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": "", "hs_analytics_source_data_1": "", "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": "", "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-01-23T15:41:56.644000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": null, "hs_num_blockers": 0, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": 0, "hs_num_decision_makers": 0, "hs_num_open_deals": 0, "hs_object_id": 5000787595, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": null, "hs_predictivecontactscore_v2": null, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.4076234698295593, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2020-12-11T01:28:27.673000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": null, "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Daxtarity", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 50, "phone": "+1 415-307-4864", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": null, "recent_deal_close_date": null, "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": null, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "website": "Daxtarity.com", "zip": "94114"}, "createdAt": "2020-12-11T01:28:27.673Z", "updatedAt": "2023-01-23T15:41:56.644Z", "archived": false, "properties_about_us": null, "properties_address": "2261 Market Street", "properties_address2": null, "properties_annualrevenue": null, "properties_city": "San Francisco", "properties_closedate": null, "properties_closedate_timestamp_earliest_value_a2a17e6e": null, "properties_country": "United States", "properties_createdate": "2020-12-11T01:28:27.673000+00:00", "properties_custom_company_property": null, "properties_days_to_close": null, "properties_description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_domain": "Daxtarity.com", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_facebook_company_page": null, "properties_facebookfans": null, "properties_first_contact_createdate": null, "properties_first_contact_createdate_timestamp_earliest_value_78b50eea": null, "properties_first_conversion_date": null, "properties_first_conversion_date_timestamp_earliest_value_61f58f2c": null, "properties_first_conversion_event_name": null, "properties_first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "properties_first_deal_created_date": null, "properties_founded_year": "2020", "properties_googleplus_page": null, "properties_hs_additional_domains": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_first_timestamp": null, "properties_hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "properties_hs_analytics_latest_source": "", "properties_hs_analytics_latest_source_data_1": "", "properties_hs_analytics_latest_source_data_2": "", "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_num_page_views": null, "properties_hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "properties_hs_analytics_num_visits": null, "properties_hs_analytics_num_visits_cardinality_sum_53d952a6": null, "properties_hs_analytics_source": "", "properties_hs_analytics_source_data_1": "", "properties_hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "properties_hs_analytics_source_data_2": "", "properties_hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "properties_hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "properties_hs_annual_revenue_currency_code": "USD", "properties_hs_avatar_filemanager_key": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": null, "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": null, "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_ideal_customer_profile": null, "properties_hs_is_target_account": null, "properties_hs_last_booked_meeting_date": null, "properties_hs_last_logged_call_date": null, "properties_hs_last_open_task_date": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": "2023-01-23T15:41:56.644000+00:00", "properties_hs_latest_createdate_of_active_subscriptions": null, "properties_hs_latest_meeting_activity": null, "properties_hs_lead_status": null, "properties_hs_merged_object_ids": null, "properties_hs_num_blockers": 0, "properties_hs_num_child_companies": 0, "properties_hs_num_contacts_with_buying_roles": 0, "properties_hs_num_decision_makers": 0, "properties_hs_num_open_deals": 0, "properties_hs_object_id": 5000787595, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_parent_company_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": null, "properties_hs_predictivecontactscore_v2": null, "properties_hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_target_account": null, "properties_hs_target_account_probability": 0.4076234698295593, "properties_hs_target_account_recommendation_snooze_time": null, "properties_hs_target_account_recommendation_state": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": null, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": null, "properties_hs_total_deal_value": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2020-12-11T01:28:27.673000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_is_public": false, "properties_lifecyclestage": null, "properties_linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "properties_linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_name": "Daxtarity", "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_associated_deals": null, "properties_num_contacted_notes": null, "properties_num_conversion_events": null, "properties_num_conversion_events_cardinality_sum_d095f14b": null, "properties_num_notes": null, "properties_numberofemployees": 50, "properties_phone": "+1 415-307-4864", "properties_recent_conversion_date": null, "properties_recent_conversion_date_timestamp_latest_value_72856da1": null, "properties_recent_conversion_event_name": null, "properties_recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_state": "CA", "properties_timezone": "America/Los_Angeles", "properties_total_money_raised": null, "properties_total_revenue": null, "properties_twitterbio": null, "properties_twitterfollowers": null, "properties_twitterhandle": "AirbyteHQ", "properties_type": null, "properties_web_technologies": "slack;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "properties_website": "Daxtarity.com", "properties_zip": "94114"}, "emitted_at": 1697714187363} +{"stream": "companies", "data": {"id": "4992593519", "properties": {"about_us": null, "address": null, "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": null, "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-10T07:58:09.554000+00:00", "custom_company_property": null, "days_to_close": null, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "airbyte.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": null, "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": "2021-05-21T10:17:06.028000+00:00", "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": null, "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_num_page_views": null, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": null, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": null, "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": "2021-05-21T10:17:28.964000+00:00", "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-01-26T11:45:49.817000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": null, "hs_num_blockers": null, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": null, "hs_num_decision_makers": null, "hs_num_open_deals": 1, "hs_object_id": 4992593519, "hs_object_source": "CONTACTS", "hs_object_source_id": "CRM_UI", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": null, "hs_predictivecontactscore_v2": null, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.5476861596107483, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": 84592236724, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2020-12-10T07:58:09.554000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": "opportunity", "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Airbyte test1", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_associated_deals": 1, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 200, "phone": "+1 415-307-4864", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": null, "recent_deal_close_date": null, "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": null, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;segment;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "website": "airbyte.io", "zip": "94114"}, "createdAt": "2020-12-10T07:58:09.554Z", "updatedAt": "2023-01-26T11:45:49.817Z", "archived": false, "properties_about_us": null, "properties_address": null, "properties_address2": null, "properties_annualrevenue": null, "properties_city": "San Francisco", "properties_closedate": null, "properties_closedate_timestamp_earliest_value_a2a17e6e": null, "properties_country": "United States", "properties_createdate": "2020-12-10T07:58:09.554000+00:00", "properties_custom_company_property": null, "properties_days_to_close": null, "properties_description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_domain": "airbyte.io", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_facebook_company_page": null, "properties_facebookfans": null, "properties_first_contact_createdate": null, "properties_first_contact_createdate_timestamp_earliest_value_78b50eea": null, "properties_first_conversion_date": null, "properties_first_conversion_date_timestamp_earliest_value_61f58f2c": null, "properties_first_conversion_event_name": null, "properties_first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "properties_first_deal_created_date": "2021-05-21T10:17:06.028000+00:00", "properties_founded_year": "2020", "properties_googleplus_page": null, "properties_hs_additional_domains": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_first_timestamp": null, "properties_hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "properties_hs_analytics_latest_source": null, "properties_hs_analytics_latest_source_data_1": null, "properties_hs_analytics_latest_source_data_2": null, "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_num_page_views": null, "properties_hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "properties_hs_analytics_num_visits": null, "properties_hs_analytics_num_visits_cardinality_sum_53d952a6": null, "properties_hs_analytics_source": null, "properties_hs_analytics_source_data_1": null, "properties_hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "properties_hs_analytics_source_data_2": null, "properties_hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "properties_hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "properties_hs_annual_revenue_currency_code": "USD", "properties_hs_avatar_filemanager_key": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": "2021-05-21T10:17:28.964000+00:00", "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": null, "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_ideal_customer_profile": null, "properties_hs_is_target_account": null, "properties_hs_last_booked_meeting_date": null, "properties_hs_last_logged_call_date": null, "properties_hs_last_open_task_date": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": "2023-01-26T11:45:49.817000+00:00", "properties_hs_latest_createdate_of_active_subscriptions": null, "properties_hs_latest_meeting_activity": null, "properties_hs_lead_status": null, "properties_hs_merged_object_ids": null, "properties_hs_num_blockers": null, "properties_hs_num_child_companies": 0, "properties_hs_num_contacts_with_buying_roles": null, "properties_hs_num_decision_makers": null, "properties_hs_num_open_deals": 1, "properties_hs_object_id": 4992593519, "properties_hs_object_source": "CONTACTS", "properties_hs_object_source_id": "CRM_UI", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_parent_company_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": null, "properties_hs_predictivecontactscore_v2": null, "properties_hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_target_account": null, "properties_hs_target_account_probability": 0.5476861596107483, "properties_hs_target_account_recommendation_snooze_time": null, "properties_hs_target_account_recommendation_state": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": 84592236724, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": null, "properties_hs_total_deal_value": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2020-12-10T07:58:09.554000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_is_public": false, "properties_lifecyclestage": "opportunity", "properties_linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "properties_linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_name": "Airbyte test1", "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_associated_deals": 1, "properties_num_contacted_notes": null, "properties_num_conversion_events": null, "properties_num_conversion_events_cardinality_sum_d095f14b": null, "properties_num_notes": null, "properties_numberofemployees": 200, "properties_phone": "+1 415-307-4864", "properties_recent_conversion_date": null, "properties_recent_conversion_date_timestamp_latest_value_72856da1": null, "properties_recent_conversion_event_name": null, "properties_recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_state": "CA", "properties_timezone": "America/Los_Angeles", "properties_total_money_raised": null, "properties_total_revenue": null, "properties_twitterbio": null, "properties_twitterfollowers": null, "properties_twitterhandle": "AirbyteHQ", "properties_type": null, "properties_web_technologies": "slack;segment;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "properties_website": "airbyte.io", "properties_zip": "94114"}, "emitted_at": 1706184485995} +{"stream": "companies", "data": {"id": "5000526215", "properties": {"about_us": null, "address": null, "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": "2023-04-04T15:00:58.081000+00:00", "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-11T01:27:40.002000+00:00", "custom_company_property": null, "days_to_close": 844, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "dataline.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": "2020-12-11T01:29:50.116000+00:00", "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": "2021-01-13T10:30:42.221000+00:00", "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": "OFFLINE", "hs_analytics_latest_source_data_1": "CONTACTS", "hs_analytics_latest_source_data_2": "CRM_UI", "hs_analytics_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "hs_analytics_num_page_views": 0, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": 0, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": "CRM_UI", "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": "2023-04-04T15:00:58.081000+00:00", "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": "2021-02-23T20:21:06.027000+00:00", "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": "2023-04-04T15:00:58.081000+00:00", "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-09-07T03:58:14.126000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": "5183403213", "hs_num_blockers": 0, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": 0, "hs_num_decision_makers": 0, "hs_num_open_deals": 2, "hs_object_id": 5000526215, "hs_object_source": "CONTACTS", "hs_object_source_id": "CRM_UI", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "companies-lifecycle-pipeline", "hs_predictivecontactscore_v2": 0.3, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.46257445216178894, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": 25564027612, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": 66508792054, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": 60010, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2020-12-11T01:27:40.002000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": "customer", "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Dataline", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 1, "num_associated_deals": 3, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 25, "phone": "", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": 60000, "recent_deal_close_date": "2023-04-04T14:59:45.103000+00:00", "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": 60000, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;segment;google_tag_manager;cloud_flare;google_analytics;intercom;lever;google_apps", "website": "dataline.io", "zip": ""}, "createdAt": "2020-12-11T01:27:40.002Z", "updatedAt": "2023-09-07T03:58:14.126Z", "archived": false, "contacts": ["151", "151"], "properties_about_us": null, "properties_address": null, "properties_address2": null, "properties_annualrevenue": null, "properties_city": "San Francisco", "properties_closedate": "2023-04-04T15:00:58.081000+00:00", "properties_closedate_timestamp_earliest_value_a2a17e6e": null, "properties_country": "United States", "properties_createdate": "2020-12-11T01:27:40.002000+00:00", "properties_custom_company_property": null, "properties_days_to_close": 844, "properties_description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_domain": "dataline.io", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_facebook_company_page": null, "properties_facebookfans": null, "properties_first_contact_createdate": "2020-12-11T01:29:50.116000+00:00", "properties_first_contact_createdate_timestamp_earliest_value_78b50eea": null, "properties_first_conversion_date": null, "properties_first_conversion_date_timestamp_earliest_value_61f58f2c": null, "properties_first_conversion_event_name": null, "properties_first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "properties_first_deal_created_date": "2021-01-13T10:30:42.221000+00:00", "properties_founded_year": "2020", "properties_googleplus_page": null, "properties_hs_additional_domains": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "properties_hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "properties_hs_analytics_latest_source": "OFFLINE", "properties_hs_analytics_latest_source_data_1": "CONTACTS", "properties_hs_analytics_latest_source_data_2": "CRM_UI", "properties_hs_analytics_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "properties_hs_analytics_num_page_views": 0, "properties_hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "properties_hs_analytics_num_visits": 0, "properties_hs_analytics_num_visits_cardinality_sum_53d952a6": null, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "CONTACTS", "properties_hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "properties_hs_analytics_source_data_2": "CRM_UI", "properties_hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "properties_hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "properties_hs_annual_revenue_currency_code": "USD", "properties_hs_avatar_filemanager_key": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_date_entered_customer": "2023-04-04T15:00:58.081000+00:00", "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": "2021-02-23T20:21:06.027000+00:00", "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": null, "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": "2023-04-04T15:00:58.081000+00:00", "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_ideal_customer_profile": null, "properties_hs_is_target_account": null, "properties_hs_last_booked_meeting_date": null, "properties_hs_last_logged_call_date": null, "properties_hs_last_open_task_date": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": "2023-09-07T03:58:14.126000+00:00", "properties_hs_latest_createdate_of_active_subscriptions": null, "properties_hs_latest_meeting_activity": null, "properties_hs_lead_status": null, "properties_hs_merged_object_ids": "5183403213", "properties_hs_num_blockers": 0, "properties_hs_num_child_companies": 0, "properties_hs_num_contacts_with_buying_roles": 0, "properties_hs_num_decision_makers": 0, "properties_hs_num_open_deals": 2, "properties_hs_object_id": 5000526215, "properties_hs_object_source": "CONTACTS", "properties_hs_object_source_id": "CRM_UI", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_parent_company_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "companies-lifecycle-pipeline", "properties_hs_predictivecontactscore_v2": 0.3, "properties_hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_target_account": null, "properties_hs_target_account_probability": 0.46257445216178894, "properties_hs_target_account_recommendation_snooze_time": null, "properties_hs_target_account_recommendation_state": null, "properties_hs_time_in_customer": 25564027612, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": 66508792054, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": null, "properties_hs_total_deal_value": 60010, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2020-12-11T01:27:40.002000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_is_public": false, "properties_lifecyclestage": "customer", "properties_linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "properties_linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_name": "Dataline", "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 1, "properties_num_associated_deals": 3, "properties_num_contacted_notes": null, "properties_num_conversion_events": null, "properties_num_conversion_events_cardinality_sum_d095f14b": null, "properties_num_notes": null, "properties_numberofemployees": 25, "properties_phone": "", "properties_recent_conversion_date": null, "properties_recent_conversion_date_timestamp_latest_value_72856da1": null, "properties_recent_conversion_event_name": null, "properties_recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "properties_recent_deal_amount": 60000, "properties_recent_deal_close_date": "2023-04-04T14:59:45.103000+00:00", "properties_state": "CA", "properties_timezone": "America/Los_Angeles", "properties_total_money_raised": null, "properties_total_revenue": 60000, "properties_twitterbio": null, "properties_twitterfollowers": null, "properties_twitterhandle": "AirbyteHQ", "properties_type": null, "properties_web_technologies": "slack;segment;google_tag_manager;cloud_flare;google_analytics;intercom;lever;google_apps", "properties_website": "dataline.io", "properties_zip": ""}, "emitted_at": 1706184485997} +{"stream": "companies", "data": {"id": "5000787595", "properties": {"about_us": null, "address": "2261 Market Street", "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": null, "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-11T01:28:27.673000+00:00", "custom_company_property": null, "days_to_close": null, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "Daxtarity.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": null, "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": null, "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": null, "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": "", "hs_analytics_latest_source_data_1": "", "hs_analytics_latest_source_data_2": "", "hs_analytics_latest_source_timestamp": null, "hs_analytics_num_page_views": null, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": null, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": "", "hs_analytics_source_data_1": "", "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": "", "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-01-23T15:41:56.644000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": null, "hs_num_blockers": 0, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": 0, "hs_num_decision_makers": 0, "hs_num_open_deals": 0, "hs_object_id": 5000787595, "hs_object_source": "CONTACTS", "hs_object_source_id": "CRM_UI", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": null, "hs_predictivecontactscore_v2": null, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.4076234698295593, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2020-12-11T01:28:27.673000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": null, "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Daxtarity", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 50, "phone": "+1 415-307-4864", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": null, "recent_deal_close_date": null, "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": null, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "website": "Daxtarity.com", "zip": "94114"}, "createdAt": "2020-12-11T01:28:27.673Z", "updatedAt": "2023-01-23T15:41:56.644Z", "archived": false, "properties_about_us": null, "properties_address": "2261 Market Street", "properties_address2": null, "properties_annualrevenue": null, "properties_city": "San Francisco", "properties_closedate": null, "properties_closedate_timestamp_earliest_value_a2a17e6e": null, "properties_country": "United States", "properties_createdate": "2020-12-11T01:28:27.673000+00:00", "properties_custom_company_property": null, "properties_days_to_close": null, "properties_description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_domain": "Daxtarity.com", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_facebook_company_page": null, "properties_facebookfans": null, "properties_first_contact_createdate": null, "properties_first_contact_createdate_timestamp_earliest_value_78b50eea": null, "properties_first_conversion_date": null, "properties_first_conversion_date_timestamp_earliest_value_61f58f2c": null, "properties_first_conversion_event_name": null, "properties_first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "properties_first_deal_created_date": null, "properties_founded_year": "2020", "properties_googleplus_page": null, "properties_hs_additional_domains": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_first_timestamp": null, "properties_hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "properties_hs_analytics_latest_source": "", "properties_hs_analytics_latest_source_data_1": "", "properties_hs_analytics_latest_source_data_2": "", "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_num_page_views": null, "properties_hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "properties_hs_analytics_num_visits": null, "properties_hs_analytics_num_visits_cardinality_sum_53d952a6": null, "properties_hs_analytics_source": "", "properties_hs_analytics_source_data_1": "", "properties_hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "properties_hs_analytics_source_data_2": "", "properties_hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "properties_hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "properties_hs_annual_revenue_currency_code": "USD", "properties_hs_avatar_filemanager_key": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": null, "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": null, "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_ideal_customer_profile": null, "properties_hs_is_target_account": null, "properties_hs_last_booked_meeting_date": null, "properties_hs_last_logged_call_date": null, "properties_hs_last_open_task_date": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": "2023-01-23T15:41:56.644000+00:00", "properties_hs_latest_createdate_of_active_subscriptions": null, "properties_hs_latest_meeting_activity": null, "properties_hs_lead_status": null, "properties_hs_merged_object_ids": null, "properties_hs_num_blockers": 0, "properties_hs_num_child_companies": 0, "properties_hs_num_contacts_with_buying_roles": 0, "properties_hs_num_decision_makers": 0, "properties_hs_num_open_deals": 0, "properties_hs_object_id": 5000787595, "properties_hs_object_source": "CONTACTS", "properties_hs_object_source_id": "CRM_UI", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_parent_company_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": null, "properties_hs_predictivecontactscore_v2": null, "properties_hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_target_account": null, "properties_hs_target_account_probability": 0.4076234698295593, "properties_hs_target_account_recommendation_snooze_time": null, "properties_hs_target_account_recommendation_state": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": null, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": null, "properties_hs_total_deal_value": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2020-12-11T01:28:27.673000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_is_public": false, "properties_lifecyclestage": null, "properties_linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "properties_linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_name": "Daxtarity", "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_associated_deals": null, "properties_num_contacted_notes": null, "properties_num_conversion_events": null, "properties_num_conversion_events_cardinality_sum_d095f14b": null, "properties_num_notes": null, "properties_numberofemployees": 50, "properties_phone": "+1 415-307-4864", "properties_recent_conversion_date": null, "properties_recent_conversion_date_timestamp_latest_value_72856da1": null, "properties_recent_conversion_event_name": null, "properties_recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_state": "CA", "properties_timezone": "America/Los_Angeles", "properties_total_money_raised": null, "properties_total_revenue": null, "properties_twitterbio": null, "properties_twitterfollowers": null, "properties_twitterhandle": "AirbyteHQ", "properties_type": null, "properties_web_technologies": "slack;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "properties_website": "Daxtarity.com", "properties_zip": "94114"}, "emitted_at": 1706184485998} {"stream": "contact_lists", "data": {"portalId": 8727216, "listId": 1, "createdAt": 1610634707370, "updatedAt": 1610634721116, "name": "tweeters", "listType": "DYNAMIC", "authorId": 0, "filters": [], "metaData": {"size": 0, "lastSizeChangeAt": 1625270400000, "processing": "DONE", "lastProcessingStateChangeAt": 1610634721950, "error": "", "listReferencesCount": null, "parentFolderId": null}, "archived": false, "teamIds": [], "ilsFilterBranch": "{\"filterBranchOperator\":\"OR\",\"filters\":[],\"filterBranches\":[{\"filterBranchOperator\":\"AND\",\"filters\":[{\"filterType\":\"PROPERTY\",\"property\":\"twitterhandle\",\"operation\":{\"propertyType\":\"string\",\"operator\":\"IS_EQUAL_TO\",\"value\":\"@hubspot\",\"defaultValue\":null,\"includeObjectsWithNoValueSet\":false,\"operationType\":\"string\",\"operatorName\":\"IS_EQUAL_TO\"},\"frameworkFilterId\":null}],\"filterBranches\":[],\"filterBranchType\":\"AND\"}],\"filterBranchType\":\"OR\"}", "readOnly": false, "dynamic": true, "internal": false, "limitExempt": false, "metaData_size": 0, "metaData_lastSizeChangeAt": 1625270400000, "metaData_processing": "DONE", "metaData_lastProcessingStateChangeAt": 1610634721950, "metaData_error": "", "metaData_listReferencesCount": null, "metaData_parentFolderId": null}, "emitted_at": 1697714189110} {"stream": "contact_lists", "data": {"portalId": 8727216, "listId": 2, "createdAt": 1610634770432, "updatedAt": 1610634780637, "name": "tweeters 1", "listType": "DYNAMIC", "authorId": 0, "filters": [], "metaData": {"size": 0, "lastSizeChangeAt": 1625270400000, "processing": "DONE", "lastProcessingStateChangeAt": 1610634781147, "error": "", "listReferencesCount": null, "parentFolderId": null}, "archived": false, "teamIds": [], "ilsFilterBranch": "{\"filterBranchOperator\":\"OR\",\"filters\":[],\"filterBranches\":[{\"filterBranchOperator\":\"AND\",\"filters\":[{\"filterType\":\"PROPERTY\",\"property\":\"twitterhandle\",\"operation\":{\"propertyType\":\"string\",\"operator\":\"IS_EQUAL_TO\",\"value\":\"@hubspot\",\"defaultValue\":null,\"includeObjectsWithNoValueSet\":false,\"operationType\":\"string\",\"operatorName\":\"IS_EQUAL_TO\"},\"frameworkFilterId\":null}],\"filterBranches\":[],\"filterBranchType\":\"AND\"}],\"filterBranchType\":\"OR\"}", "readOnly": false, "dynamic": true, "internal": false, "limitExempt": false, "metaData_size": 0, "metaData_lastSizeChangeAt": 1625270400000, "metaData_processing": "DONE", "metaData_lastProcessingStateChangeAt": 1610634781147, "metaData_error": "", "metaData_listReferencesCount": null, "metaData_parentFolderId": null}, "emitted_at": 1697714189112} {"stream": "contact_lists", "data": {"portalId": 8727216, "listId": 3, "createdAt": 1610634774356, "updatedAt": 1610634787734, "name": "tweeters 2", "listType": "DYNAMIC", "authorId": 0, "filters": [], "metaData": {"size": 0, "lastSizeChangeAt": 1625270400000, "processing": "DONE", "lastProcessingStateChangeAt": 1610634788528, "error": "", "listReferencesCount": null, "parentFolderId": null}, "archived": false, "teamIds": [], "ilsFilterBranch": "{\"filterBranchOperator\":\"OR\",\"filters\":[],\"filterBranches\":[{\"filterBranchOperator\":\"AND\",\"filters\":[{\"filterType\":\"PROPERTY\",\"property\":\"twitterhandle\",\"operation\":{\"propertyType\":\"string\",\"operator\":\"IS_EQUAL_TO\",\"value\":\"@hubspot\",\"defaultValue\":null,\"includeObjectsWithNoValueSet\":false,\"operationType\":\"string\",\"operatorName\":\"IS_EQUAL_TO\"},\"frameworkFilterId\":null}],\"filterBranches\":[],\"filterBranchType\":\"AND\"}],\"filterBranchType\":\"OR\"}", "readOnly": false, "dynamic": true, "internal": false, "limitExempt": false, "metaData_size": 0, "metaData_lastSizeChangeAt": 1625270400000, "metaData_processing": "DONE", "metaData_lastProcessingStateChangeAt": 1610634788528, "metaData_error": "", "metaData_listReferencesCount": null, "metaData_parentFolderId": null}, "emitted_at": 1697714189113} -{"stream": "contacts", "data": {"id": "151", "properties": {"address": null, "annualrevenue": null, "associatedcompanyid": 5000526215, "associatedcompanylastupdated": null, "city": null, "closedate": null, "company": null, "company_size": null, "country": null, "createdate": "2020-12-11T01:29:50.116000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "shef@dne.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "she", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "151", "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_2": "CRM_UI", "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": 1, "hs_count_is_worked": 0, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "dne.io", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_disqualified_lead_date": null, "hs_latest_meeting_activity": null, "hs_latest_open_lead_date": null, "hs_latest_qualified_lead_date": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "CONTACTS", "hs_latest_source_data_2": "CRM_UI", "hs_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": null, "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": "2020-12-11T01:29:50.116000+00:00", "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 151, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.3, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_3", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": null, "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": 94172907549, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_v2_cumulative_time_in_customer": null, "hs_v2_cumulative_time_in_evangelist": null, "hs_v2_cumulative_time_in_lead": null, "hs_v2_cumulative_time_in_marketingqualifiedlead": null, "hs_v2_cumulative_time_in_opportunity": null, "hs_v2_cumulative_time_in_other": null, "hs_v2_cumulative_time_in_salesqualifiedlead": null, "hs_v2_cumulative_time_in_subscriber": null, "hs_v2_date_entered_customer": null, "hs_v2_date_entered_evangelist": null, "hs_v2_date_entered_lead": null, "hs_v2_date_entered_marketingqualifiedlead": null, "hs_v2_date_entered_opportunity": null, "hs_v2_date_entered_other": null, "hs_v2_date_entered_salesqualifiedlead": null, "hs_v2_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "hs_v2_date_exited_customer": null, "hs_v2_date_exited_evangelist": null, "hs_v2_date_exited_lead": null, "hs_v2_date_exited_marketingqualifiedlead": null, "hs_v2_date_exited_opportunity": null, "hs_v2_date_exited_other": null, "hs_v2_date_exited_salesqualifiedlead": null, "hs_v2_date_exited_subscriber": null, "hs_v2_latest_time_in_customer": null, "hs_v2_latest_time_in_evangelist": null, "hs_v2_latest_time_in_lead": null, "hs_v2_latest_time_in_marketingqualifiedlead": null, "hs_v2_latest_time_in_opportunity": null, "hs_v2_latest_time_in_other": null, "hs_v2_latest_time_in_salesqualifiedlead": null, "hs_v2_latest_time_in_subscriber": null, "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": "2020-12-11T01:29:50.093000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-11-22T21:10:04.346000+00:00", "lastname": "nad", "lifecyclestage": "subscriber", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": null, "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": null, "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": null, "work_email": null, "zip": null}, "createdAt": "2020-12-11T01:29:50.116Z", "updatedAt": "2023-11-22T21:10:04.346Z", "archived": false, "companies": ["5000526215", "5000526215"], "properties_address": null, "properties_annualrevenue": null, "properties_associatedcompanyid": 5000526215, "properties_associatedcompanylastupdated": null, "properties_city": null, "properties_closedate": null, "properties_company": null, "properties_company_size": null, "properties_country": null, "properties_createdate": "2020-12-11T01:29:50.116000+00:00", "properties_currentlyinworkflow": null, "properties_date_of_birth": null, "properties_days_to_close": null, "properties_degree": null, "properties_email": "shef@dne.io", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_fax": null, "properties_field_of_study": null, "properties_first_conversion_date": null, "properties_first_conversion_event_name": null, "properties_first_deal_created_date": null, "properties_firstname": "she", "properties_gender": null, "properties_graduation_date": null, "properties_hs_additional_emails": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_contact_vids": "151", "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_average_page_views": 0, "properties_hs_analytics_first_referrer": null, "properties_hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_url": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_last_referrer": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_url": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_num_event_completions": 0, "properties_hs_analytics_num_page_views": 0, "properties_hs_analytics_num_visits": 0, "properties_hs_analytics_revenue": 0.0, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "CONTACTS", "properties_hs_analytics_source_data_2": "CRM_UI", "properties_hs_avatar_filemanager_key": null, "properties_hs_buying_role": null, "properties_hs_calculated_form_submissions": null, "properties_hs_calculated_merged_vids": null, "properties_hs_calculated_mobile_number": null, "properties_hs_calculated_phone_number": null, "properties_hs_calculated_phone_number_area_code": null, "properties_hs_calculated_phone_number_country_code": null, "properties_hs_calculated_phone_number_region_code": null, "properties_hs_clicked_linkedin_ad": null, "properties_hs_content_membership_email": null, "properties_hs_content_membership_email_confirmed": null, "properties_hs_content_membership_notes": null, "properties_hs_content_membership_registered_at": null, "properties_hs_content_membership_registration_domain_sent_to": null, "properties_hs_content_membership_registration_email_sent_at": null, "properties_hs_content_membership_status": null, "properties_hs_conversations_visitor_email": null, "properties_hs_count_is_unworked": 1, "properties_hs_count_is_worked": 0, "properties_hs_created_by_conversations": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": null, "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_document_last_revisited": null, "properties_hs_email_bad_address": null, "properties_hs_email_bounce": null, "properties_hs_email_click": null, "properties_hs_email_customer_quarantined_reason": null, "properties_hs_email_delivered": null, "properties_hs_email_domain": "dne.io", "properties_hs_email_first_click_date": null, "properties_hs_email_first_open_date": null, "properties_hs_email_first_reply_date": null, "properties_hs_email_first_send_date": null, "properties_hs_email_hard_bounce_reason": null, "properties_hs_email_hard_bounce_reason_enum": null, "properties_hs_email_is_ineligible": null, "properties_hs_email_last_click_date": null, "properties_hs_email_last_email_name": null, "properties_hs_email_last_open_date": null, "properties_hs_email_last_reply_date": null, "properties_hs_email_last_send_date": null, "properties_hs_email_open": null, "properties_hs_email_optout": null, "properties_hs_email_optout_10798197": null, "properties_hs_email_optout_11890603": null, "properties_hs_email_optout_11890831": null, "properties_hs_email_optout_23704464": null, "properties_hs_email_optout_94692364": null, "properties_hs_email_quarantined": null, "properties_hs_email_quarantined_reason": null, "properties_hs_email_recipient_fatigue_recovery_time": null, "properties_hs_email_replied": null, "properties_hs_email_sends_since_last_engagement": null, "properties_hs_emailconfirmationstatus": null, "properties_hs_facebook_ad_clicked": null, "properties_hs_facebook_click_id": null, "properties_hs_feedback_last_nps_follow_up": null, "properties_hs_feedback_last_nps_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_feedback_show_nps_web_survey": null, "properties_hs_first_engagement_object_id": null, "properties_hs_first_outreach_date": null, "properties_hs_first_subscription_create_date": null, "properties_hs_google_click_id": null, "properties_hs_has_active_subscription": null, "properties_hs_ip_timezone": null, "properties_hs_is_contact": true, "properties_hs_is_unworked": true, "properties_hs_language": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": null, "properties_hs_latest_disqualified_lead_date": null, "properties_hs_latest_meeting_activity": null, "properties_hs_latest_open_lead_date": null, "properties_hs_latest_qualified_lead_date": null, "properties_hs_latest_sequence_ended_date": null, "properties_hs_latest_sequence_enrolled": null, "properties_hs_latest_sequence_enrolled_date": null, "properties_hs_latest_sequence_finished_date": null, "properties_hs_latest_sequence_unenrolled_date": null, "properties_hs_latest_source": "OFFLINE", "properties_hs_latest_source_data_1": "CONTACTS", "properties_hs_latest_source_data_2": "CRM_UI", "properties_hs_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "properties_hs_latest_subscription_create_date": null, "properties_hs_lead_status": null, "properties_hs_legal_basis": null, "properties_hs_lifecyclestage_customer_date": null, "properties_hs_lifecyclestage_evangelist_date": null, "properties_hs_lifecyclestage_lead_date": null, "properties_hs_lifecyclestage_marketingqualifiedlead_date": null, "properties_hs_lifecyclestage_opportunity_date": null, "properties_hs_lifecyclestage_other_date": null, "properties_hs_lifecyclestage_salesqualifiedlead_date": null, "properties_hs_lifecyclestage_subscriber_date": "2020-12-11T01:29:50.116000+00:00", "properties_hs_linkedin_ad_clicked": null, "properties_hs_marketable_reason_id": null, "properties_hs_marketable_reason_type": null, "properties_hs_marketable_status": "false", "properties_hs_marketable_until_renewal": "false", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 151, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_persona": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "contacts-lifecycle-pipeline", "properties_hs_predictivecontactscore": null, "properties_hs_predictivecontactscore_v2": 0.3, "properties_hs_predictivecontactscorebucket": null, "properties_hs_predictivescoringtier": "tier_3", "properties_hs_read_only": null, "properties_hs_sa_first_engagement_date": null, "properties_hs_sa_first_engagement_descr": null, "properties_hs_sa_first_engagement_object_type": null, "properties_hs_sales_email_last_clicked": null, "properties_hs_sales_email_last_opened": null, "properties_hs_sales_email_last_replied": null, "properties_hs_searchable_calculated_international_mobile_number": null, "properties_hs_searchable_calculated_international_phone_number": null, "properties_hs_searchable_calculated_mobile_number": null, "properties_hs_searchable_calculated_phone_number": null, "properties_hs_sequences_actively_enrolled_count": null, "properties_hs_sequences_enrolled_count": null, "properties_hs_sequences_is_enrolled": null, "properties_hs_testpurge": null, "properties_hs_testrollback": null, "properties_hs_time_between_contact_creation_and_deal_close": null, "properties_hs_time_between_contact_creation_and_deal_creation": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": null, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": 94172907549, "properties_hs_time_to_first_engagement": null, "properties_hs_time_to_move_from_lead_to_customer": null, "properties_hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_opportunity_to_customer": null, "properties_hs_time_to_move_from_salesqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_subscriber_to_customer": null, "properties_hs_timezone": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_v2_cumulative_time_in_customer": null, "properties_hs_v2_cumulative_time_in_evangelist": null, "properties_hs_v2_cumulative_time_in_lead": null, "properties_hs_v2_cumulative_time_in_marketingqualifiedlead": null, "properties_hs_v2_cumulative_time_in_opportunity": null, "properties_hs_v2_cumulative_time_in_other": null, "properties_hs_v2_cumulative_time_in_salesqualifiedlead": null, "properties_hs_v2_cumulative_time_in_subscriber": null, "properties_hs_v2_date_entered_customer": null, "properties_hs_v2_date_entered_evangelist": null, "properties_hs_v2_date_entered_lead": null, "properties_hs_v2_date_entered_marketingqualifiedlead": null, "properties_hs_v2_date_entered_opportunity": null, "properties_hs_v2_date_entered_other": null, "properties_hs_v2_date_entered_salesqualifiedlead": null, "properties_hs_v2_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "properties_hs_v2_date_exited_customer": null, "properties_hs_v2_date_exited_evangelist": null, "properties_hs_v2_date_exited_lead": null, "properties_hs_v2_date_exited_marketingqualifiedlead": null, "properties_hs_v2_date_exited_opportunity": null, "properties_hs_v2_date_exited_other": null, "properties_hs_v2_date_exited_salesqualifiedlead": null, "properties_hs_v2_date_exited_subscriber": null, "properties_hs_v2_latest_time_in_customer": null, "properties_hs_v2_latest_time_in_evangelist": null, "properties_hs_v2_latest_time_in_lead": null, "properties_hs_v2_latest_time_in_marketingqualifiedlead": null, "properties_hs_v2_latest_time_in_opportunity": null, "properties_hs_v2_latest_time_in_other": null, "properties_hs_v2_latest_time_in_salesqualifiedlead": null, "properties_hs_v2_latest_time_in_subscriber": null, "properties_hs_was_imported": null, "properties_hs_whatsapp_phone_number": null, "properties_hubspot_owner_assigneddate": "2020-12-11T01:29:50.093000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_ip_city": null, "properties_ip_country": null, "properties_ip_country_code": null, "properties_ip_latlon": null, "properties_ip_state": null, "properties_ip_state_code": null, "properties_ip_zipcode": null, "properties_job_function": null, "properties_jobtitle": null, "properties_lastmodifieddate": "2023-11-22T21:10:04.346000+00:00", "properties_lastname": "nad", "properties_lifecyclestage": "subscriber", "properties_marital_status": null, "properties_message": null, "properties_military_status": null, "properties_mobilephone": null, "properties_my_custom_test_property": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_deals": null, "properties_num_contacted_notes": null, "properties_num_conversion_events": 0, "properties_num_notes": null, "properties_num_unique_conversion_events": 0, "properties_numemployees": null, "properties_phone": null, "properties_recent_conversion_date": null, "properties_recent_conversion_event_name": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_relationship_status": null, "properties_salutation": null, "properties_school": null, "properties_seniority": null, "properties_start_date": null, "properties_state": null, "properties_surveymonkeyeventlastupdated": null, "properties_test": null, "properties_total_revenue": null, "properties_twitterhandle": null, "properties_webinareventlastupdated": null, "properties_website": null, "properties_work_email": null, "properties_zip": null}, "emitted_at": 1701823098427} -{"stream": "contacts", "data": {"id": "251", "properties": {"address": "25000000 First Street", "annualrevenue": null, "associatedcompanyid": 5170561229, "associatedcompanylastupdated": null, "city": "Cambridge", "closedate": null, "company": "HubSpot", "company_size": null, "country": "USA", "createdate": "2021-02-22T14:05:09.944000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "testingdsapis@hubspot.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "Test User 5001", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "251", "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2021-02-22T14:05:09.944000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "API", "hs_analytics_source_data_2": null, "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": null, "hs_count_is_worked": null, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "hubspot.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_disqualified_lead_date": null, "hs_latest_meeting_activity": null, "hs_latest_open_lead_date": null, "hs_latest_qualified_lead_date": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "API", "hs_latest_source_data_2": null, "hs_latest_source_timestamp": "2021-02-22T14:05:10.036000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": null, "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": "2021-02-22T14:05:09.944000+00:00", "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 251, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.29, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_4", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "5551222323", "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": 87820387720, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_v2_cumulative_time_in_customer": null, "hs_v2_cumulative_time_in_evangelist": null, "hs_v2_cumulative_time_in_lead": null, "hs_v2_cumulative_time_in_marketingqualifiedlead": null, "hs_v2_cumulative_time_in_opportunity": null, "hs_v2_cumulative_time_in_other": null, "hs_v2_cumulative_time_in_salesqualifiedlead": null, "hs_v2_cumulative_time_in_subscriber": null, "hs_v2_date_entered_customer": null, "hs_v2_date_entered_evangelist": null, "hs_v2_date_entered_lead": null, "hs_v2_date_entered_marketingqualifiedlead": null, "hs_v2_date_entered_opportunity": null, "hs_v2_date_entered_other": null, "hs_v2_date_entered_salesqualifiedlead": null, "hs_v2_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "hs_v2_date_exited_customer": null, "hs_v2_date_exited_evangelist": null, "hs_v2_date_exited_lead": null, "hs_v2_date_exited_marketingqualifiedlead": null, "hs_v2_date_exited_opportunity": null, "hs_v2_date_exited_other": null, "hs_v2_date_exited_salesqualifiedlead": null, "hs_v2_date_exited_subscriber": null, "hs_v2_latest_time_in_customer": null, "hs_v2_latest_time_in_evangelist": null, "hs_v2_latest_time_in_lead": null, "hs_v2_latest_time_in_marketingqualifiedlead": null, "hs_v2_latest_time_in_opportunity": null, "hs_v2_latest_time_in_other": null, "hs_v2_latest_time_in_salesqualifiedlead": null, "hs_v2_latest_time_in_subscriber": null, "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-03-21T19:29:13.036000+00:00", "lastname": "Test Lastname 5001", "lifecyclestage": "subscriber", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "555-122-2323", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": "MA", "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": "http://hubspot.com", "work_email": null, "zip": "02139"}, "createdAt": "2021-02-22T14:05:09.944Z", "updatedAt": "2023-03-21T19:29:13.036Z", "archived": false, "companies": ["5170561229", "5170561229"], "properties_address": "25000000 First Street", "properties_annualrevenue": null, "properties_associatedcompanyid": 5170561229, "properties_associatedcompanylastupdated": null, "properties_city": "Cambridge", "properties_closedate": null, "properties_company": "HubSpot", "properties_company_size": null, "properties_country": "USA", "properties_createdate": "2021-02-22T14:05:09.944000+00:00", "properties_currentlyinworkflow": null, "properties_date_of_birth": null, "properties_days_to_close": null, "properties_degree": null, "properties_email": "testingdsapis@hubspot.com", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_fax": null, "properties_field_of_study": null, "properties_first_conversion_date": null, "properties_first_conversion_event_name": null, "properties_first_deal_created_date": null, "properties_firstname": "Test User 5001", "properties_gender": null, "properties_graduation_date": null, "properties_hs_additional_emails": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_contact_vids": "251", "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_analytics_average_page_views": 0, "properties_hs_analytics_first_referrer": null, "properties_hs_analytics_first_timestamp": "2021-02-22T14:05:09.944000+00:00", "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_url": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_last_referrer": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_url": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_num_event_completions": 0, "properties_hs_analytics_num_page_views": 0, "properties_hs_analytics_num_visits": 0, "properties_hs_analytics_revenue": 0.0, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "API", "properties_hs_analytics_source_data_2": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_buying_role": null, "properties_hs_calculated_form_submissions": null, "properties_hs_calculated_merged_vids": null, "properties_hs_calculated_mobile_number": null, "properties_hs_calculated_phone_number": null, "properties_hs_calculated_phone_number_area_code": null, "properties_hs_calculated_phone_number_country_code": null, "properties_hs_calculated_phone_number_region_code": null, "properties_hs_clicked_linkedin_ad": null, "properties_hs_content_membership_email": null, "properties_hs_content_membership_email_confirmed": null, "properties_hs_content_membership_notes": null, "properties_hs_content_membership_registered_at": null, "properties_hs_content_membership_registration_domain_sent_to": null, "properties_hs_content_membership_registration_email_sent_at": null, "properties_hs_content_membership_status": null, "properties_hs_conversations_visitor_email": null, "properties_hs_count_is_unworked": null, "properties_hs_count_is_worked": null, "properties_hs_created_by_conversations": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": null, "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_document_last_revisited": null, "properties_hs_email_bad_address": null, "properties_hs_email_bounce": null, "properties_hs_email_click": null, "properties_hs_email_customer_quarantined_reason": null, "properties_hs_email_delivered": null, "properties_hs_email_domain": "hubspot.com", "properties_hs_email_first_click_date": null, "properties_hs_email_first_open_date": null, "properties_hs_email_first_reply_date": null, "properties_hs_email_first_send_date": null, "properties_hs_email_hard_bounce_reason": null, "properties_hs_email_hard_bounce_reason_enum": null, "properties_hs_email_is_ineligible": null, "properties_hs_email_last_click_date": null, "properties_hs_email_last_email_name": null, "properties_hs_email_last_open_date": null, "properties_hs_email_last_reply_date": null, "properties_hs_email_last_send_date": null, "properties_hs_email_open": null, "properties_hs_email_optout": null, "properties_hs_email_optout_10798197": null, "properties_hs_email_optout_11890603": null, "properties_hs_email_optout_11890831": null, "properties_hs_email_optout_23704464": null, "properties_hs_email_optout_94692364": null, "properties_hs_email_quarantined": null, "properties_hs_email_quarantined_reason": null, "properties_hs_email_recipient_fatigue_recovery_time": null, "properties_hs_email_replied": null, "properties_hs_email_sends_since_last_engagement": null, "properties_hs_emailconfirmationstatus": null, "properties_hs_facebook_ad_clicked": null, "properties_hs_facebook_click_id": null, "properties_hs_feedback_last_nps_follow_up": null, "properties_hs_feedback_last_nps_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_feedback_show_nps_web_survey": null, "properties_hs_first_engagement_object_id": null, "properties_hs_first_outreach_date": null, "properties_hs_first_subscription_create_date": null, "properties_hs_google_click_id": null, "properties_hs_has_active_subscription": null, "properties_hs_ip_timezone": null, "properties_hs_is_contact": true, "properties_hs_is_unworked": true, "properties_hs_language": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": null, "properties_hs_latest_disqualified_lead_date": null, "properties_hs_latest_meeting_activity": null, "properties_hs_latest_open_lead_date": null, "properties_hs_latest_qualified_lead_date": null, "properties_hs_latest_sequence_ended_date": null, "properties_hs_latest_sequence_enrolled": null, "properties_hs_latest_sequence_enrolled_date": null, "properties_hs_latest_sequence_finished_date": null, "properties_hs_latest_sequence_unenrolled_date": null, "properties_hs_latest_source": "OFFLINE", "properties_hs_latest_source_data_1": "API", "properties_hs_latest_source_data_2": null, "properties_hs_latest_source_timestamp": "2021-02-22T14:05:10.036000+00:00", "properties_hs_latest_subscription_create_date": null, "properties_hs_lead_status": null, "properties_hs_legal_basis": null, "properties_hs_lifecyclestage_customer_date": null, "properties_hs_lifecyclestage_evangelist_date": null, "properties_hs_lifecyclestage_lead_date": null, "properties_hs_lifecyclestage_marketingqualifiedlead_date": null, "properties_hs_lifecyclestage_opportunity_date": null, "properties_hs_lifecyclestage_other_date": null, "properties_hs_lifecyclestage_salesqualifiedlead_date": null, "properties_hs_lifecyclestage_subscriber_date": "2021-02-22T14:05:09.944000+00:00", "properties_hs_linkedin_ad_clicked": null, "properties_hs_marketable_reason_id": null, "properties_hs_marketable_reason_type": null, "properties_hs_marketable_status": "false", "properties_hs_marketable_until_renewal": "false", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 251, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_persona": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "contacts-lifecycle-pipeline", "properties_hs_predictivecontactscore": null, "properties_hs_predictivecontactscore_v2": 0.29, "properties_hs_predictivecontactscorebucket": null, "properties_hs_predictivescoringtier": "tier_4", "properties_hs_read_only": null, "properties_hs_sa_first_engagement_date": null, "properties_hs_sa_first_engagement_descr": null, "properties_hs_sa_first_engagement_object_type": null, "properties_hs_sales_email_last_clicked": null, "properties_hs_sales_email_last_opened": null, "properties_hs_sales_email_last_replied": null, "properties_hs_searchable_calculated_international_mobile_number": null, "properties_hs_searchable_calculated_international_phone_number": null, "properties_hs_searchable_calculated_mobile_number": null, "properties_hs_searchable_calculated_phone_number": "5551222323", "properties_hs_sequences_actively_enrolled_count": null, "properties_hs_sequences_enrolled_count": null, "properties_hs_sequences_is_enrolled": null, "properties_hs_testpurge": null, "properties_hs_testrollback": null, "properties_hs_time_between_contact_creation_and_deal_close": null, "properties_hs_time_between_contact_creation_and_deal_creation": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": null, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": 87820387720, "properties_hs_time_to_first_engagement": null, "properties_hs_time_to_move_from_lead_to_customer": null, "properties_hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_opportunity_to_customer": null, "properties_hs_time_to_move_from_salesqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_subscriber_to_customer": null, "properties_hs_timezone": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_v2_cumulative_time_in_customer": null, "properties_hs_v2_cumulative_time_in_evangelist": null, "properties_hs_v2_cumulative_time_in_lead": null, "properties_hs_v2_cumulative_time_in_marketingqualifiedlead": null, "properties_hs_v2_cumulative_time_in_opportunity": null, "properties_hs_v2_cumulative_time_in_other": null, "properties_hs_v2_cumulative_time_in_salesqualifiedlead": null, "properties_hs_v2_cumulative_time_in_subscriber": null, "properties_hs_v2_date_entered_customer": null, "properties_hs_v2_date_entered_evangelist": null, "properties_hs_v2_date_entered_lead": null, "properties_hs_v2_date_entered_marketingqualifiedlead": null, "properties_hs_v2_date_entered_opportunity": null, "properties_hs_v2_date_entered_other": null, "properties_hs_v2_date_entered_salesqualifiedlead": null, "properties_hs_v2_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "properties_hs_v2_date_exited_customer": null, "properties_hs_v2_date_exited_evangelist": null, "properties_hs_v2_date_exited_lead": null, "properties_hs_v2_date_exited_marketingqualifiedlead": null, "properties_hs_v2_date_exited_opportunity": null, "properties_hs_v2_date_exited_other": null, "properties_hs_v2_date_exited_salesqualifiedlead": null, "properties_hs_v2_date_exited_subscriber": null, "properties_hs_v2_latest_time_in_customer": null, "properties_hs_v2_latest_time_in_evangelist": null, "properties_hs_v2_latest_time_in_lead": null, "properties_hs_v2_latest_time_in_marketingqualifiedlead": null, "properties_hs_v2_latest_time_in_opportunity": null, "properties_hs_v2_latest_time_in_other": null, "properties_hs_v2_latest_time_in_salesqualifiedlead": null, "properties_hs_v2_latest_time_in_subscriber": null, "properties_hs_was_imported": null, "properties_hs_whatsapp_phone_number": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_ip_city": null, "properties_ip_country": null, "properties_ip_country_code": null, "properties_ip_latlon": null, "properties_ip_state": null, "properties_ip_state_code": null, "properties_ip_zipcode": null, "properties_job_function": null, "properties_jobtitle": null, "properties_lastmodifieddate": "2023-03-21T19:29:13.036000+00:00", "properties_lastname": "Test Lastname 5001", "properties_lifecyclestage": "subscriber", "properties_marital_status": null, "properties_message": null, "properties_military_status": null, "properties_mobilephone": null, "properties_my_custom_test_property": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_deals": null, "properties_num_contacted_notes": null, "properties_num_conversion_events": 0, "properties_num_notes": null, "properties_num_unique_conversion_events": 0, "properties_numemployees": null, "properties_phone": "555-122-2323", "properties_recent_conversion_date": null, "properties_recent_conversion_event_name": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_relationship_status": null, "properties_salutation": null, "properties_school": null, "properties_seniority": null, "properties_start_date": null, "properties_state": "MA", "properties_surveymonkeyeventlastupdated": null, "properties_test": null, "properties_total_revenue": null, "properties_twitterhandle": null, "properties_webinareventlastupdated": null, "properties_website": "http://hubspot.com", "properties_work_email": null, "properties_zip": "02139"}, "emitted_at": 1701823098430} -{"stream": "contacts", "data": {"id": "401", "properties": {"address": "25 First Street", "annualrevenue": null, "associatedcompanyid": null, "associatedcompanylastupdated": null, "city": "Cambridge", "closedate": null, "company": null, "company_size": null, "country": null, "createdate": "2021-02-23T20:10:36.191000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "macmitch@hubspot.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "Mac", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "401", "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2021-02-23T20:10:36.181000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "IMPORT", "hs_analytics_source_data_2": "13256565", "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": "+18884827768", "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": "US", "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": 1, "hs_count_is_worked": 0, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "hubspot.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": "OTHER", "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_disqualified_lead_date": null, "hs_latest_meeting_activity": null, "hs_latest_open_lead_date": null, "hs_latest_qualified_lead_date": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "IMPORT", "hs_latest_source_data_2": "13256565", "hs_latest_source_timestamp": "2021-02-23T20:10:36.210000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": "2021-02-23T20:10:36.181000+00:00", "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": null, "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 401, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.29, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_4", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "8884827768", "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": 87712061483, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_v2_cumulative_time_in_customer": null, "hs_v2_cumulative_time_in_evangelist": null, "hs_v2_cumulative_time_in_lead": null, "hs_v2_cumulative_time_in_marketingqualifiedlead": null, "hs_v2_cumulative_time_in_opportunity": null, "hs_v2_cumulative_time_in_other": null, "hs_v2_cumulative_time_in_salesqualifiedlead": null, "hs_v2_cumulative_time_in_subscriber": null, "hs_v2_date_entered_customer": null, "hs_v2_date_entered_evangelist": null, "hs_v2_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "hs_v2_date_entered_marketingqualifiedlead": null, "hs_v2_date_entered_opportunity": null, "hs_v2_date_entered_other": null, "hs_v2_date_entered_salesqualifiedlead": null, "hs_v2_date_entered_subscriber": null, "hs_v2_date_exited_customer": null, "hs_v2_date_exited_evangelist": null, "hs_v2_date_exited_lead": null, "hs_v2_date_exited_marketingqualifiedlead": null, "hs_v2_date_exited_opportunity": null, "hs_v2_date_exited_other": null, "hs_v2_date_exited_salesqualifiedlead": null, "hs_v2_date_exited_subscriber": null, "hs_v2_latest_time_in_customer": null, "hs_v2_latest_time_in_evangelist": null, "hs_v2_latest_time_in_lead": null, "hs_v2_latest_time_in_marketingqualifiedlead": null, "hs_v2_latest_time_in_opportunity": null, "hs_v2_latest_time_in_other": null, "hs_v2_latest_time_in_salesqualifiedlead": null, "hs_v2_latest_time_in_subscriber": null, "hs_was_imported": true, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": "2021-05-21T10:20:30.963000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-03-21T19:31:00.563000+00:00", "lastname": "Mitchell", "lifecyclestage": "lead", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "1(888) 482-7768", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": "MA", "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": null, "work_email": null, "zip": "21430"}, "createdAt": "2021-02-23T20:10:36.191Z", "updatedAt": "2023-03-21T19:31:00.563Z", "archived": false, "properties_address": "25 First Street", "properties_annualrevenue": null, "properties_associatedcompanyid": null, "properties_associatedcompanylastupdated": null, "properties_city": "Cambridge", "properties_closedate": null, "properties_company": null, "properties_company_size": null, "properties_country": null, "properties_createdate": "2021-02-23T20:10:36.191000+00:00", "properties_currentlyinworkflow": null, "properties_date_of_birth": null, "properties_days_to_close": null, "properties_degree": null, "properties_email": "macmitch@hubspot.com", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_fax": null, "properties_field_of_study": null, "properties_first_conversion_date": null, "properties_first_conversion_event_name": null, "properties_first_deal_created_date": null, "properties_firstname": "Mac", "properties_gender": null, "properties_graduation_date": null, "properties_hs_additional_emails": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_contact_vids": "401", "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_average_page_views": 0, "properties_hs_analytics_first_referrer": null, "properties_hs_analytics_first_timestamp": "2021-02-23T20:10:36.181000+00:00", "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_url": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_last_referrer": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_url": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_num_event_completions": 0, "properties_hs_analytics_num_page_views": 0, "properties_hs_analytics_num_visits": 0, "properties_hs_analytics_revenue": 0.0, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "IMPORT", "properties_hs_analytics_source_data_2": "13256565", "properties_hs_avatar_filemanager_key": null, "properties_hs_buying_role": null, "properties_hs_calculated_form_submissions": null, "properties_hs_calculated_merged_vids": null, "properties_hs_calculated_mobile_number": null, "properties_hs_calculated_phone_number": "+18884827768", "properties_hs_calculated_phone_number_area_code": null, "properties_hs_calculated_phone_number_country_code": "US", "properties_hs_calculated_phone_number_region_code": null, "properties_hs_clicked_linkedin_ad": null, "properties_hs_content_membership_email": null, "properties_hs_content_membership_email_confirmed": null, "properties_hs_content_membership_notes": null, "properties_hs_content_membership_registered_at": null, "properties_hs_content_membership_registration_domain_sent_to": null, "properties_hs_content_membership_registration_email_sent_at": null, "properties_hs_content_membership_status": null, "properties_hs_conversations_visitor_email": null, "properties_hs_count_is_unworked": 1, "properties_hs_count_is_worked": 0, "properties_hs_created_by_conversations": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": null, "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": null, "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_document_last_revisited": null, "properties_hs_email_bad_address": null, "properties_hs_email_bounce": null, "properties_hs_email_click": null, "properties_hs_email_customer_quarantined_reason": null, "properties_hs_email_delivered": null, "properties_hs_email_domain": "hubspot.com", "properties_hs_email_first_click_date": null, "properties_hs_email_first_open_date": null, "properties_hs_email_first_reply_date": null, "properties_hs_email_first_send_date": null, "properties_hs_email_hard_bounce_reason": null, "properties_hs_email_hard_bounce_reason_enum": "OTHER", "properties_hs_email_is_ineligible": null, "properties_hs_email_last_click_date": null, "properties_hs_email_last_email_name": null, "properties_hs_email_last_open_date": null, "properties_hs_email_last_reply_date": null, "properties_hs_email_last_send_date": null, "properties_hs_email_open": null, "properties_hs_email_optout": null, "properties_hs_email_optout_10798197": null, "properties_hs_email_optout_11890603": null, "properties_hs_email_optout_11890831": null, "properties_hs_email_optout_23704464": null, "properties_hs_email_optout_94692364": null, "properties_hs_email_quarantined": null, "properties_hs_email_quarantined_reason": null, "properties_hs_email_recipient_fatigue_recovery_time": null, "properties_hs_email_replied": null, "properties_hs_email_sends_since_last_engagement": null, "properties_hs_emailconfirmationstatus": null, "properties_hs_facebook_ad_clicked": null, "properties_hs_facebook_click_id": null, "properties_hs_feedback_last_nps_follow_up": null, "properties_hs_feedback_last_nps_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_feedback_show_nps_web_survey": null, "properties_hs_first_engagement_object_id": null, "properties_hs_first_outreach_date": null, "properties_hs_first_subscription_create_date": null, "properties_hs_google_click_id": null, "properties_hs_has_active_subscription": null, "properties_hs_ip_timezone": null, "properties_hs_is_contact": true, "properties_hs_is_unworked": true, "properties_hs_language": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": null, "properties_hs_latest_disqualified_lead_date": null, "properties_hs_latest_meeting_activity": null, "properties_hs_latest_open_lead_date": null, "properties_hs_latest_qualified_lead_date": null, "properties_hs_latest_sequence_ended_date": null, "properties_hs_latest_sequence_enrolled": null, "properties_hs_latest_sequence_enrolled_date": null, "properties_hs_latest_sequence_finished_date": null, "properties_hs_latest_sequence_unenrolled_date": null, "properties_hs_latest_source": "OFFLINE", "properties_hs_latest_source_data_1": "IMPORT", "properties_hs_latest_source_data_2": "13256565", "properties_hs_latest_source_timestamp": "2021-02-23T20:10:36.210000+00:00", "properties_hs_latest_subscription_create_date": null, "properties_hs_lead_status": null, "properties_hs_legal_basis": null, "properties_hs_lifecyclestage_customer_date": null, "properties_hs_lifecyclestage_evangelist_date": null, "properties_hs_lifecyclestage_lead_date": "2021-02-23T20:10:36.181000+00:00", "properties_hs_lifecyclestage_marketingqualifiedlead_date": null, "properties_hs_lifecyclestage_opportunity_date": null, "properties_hs_lifecyclestage_other_date": null, "properties_hs_lifecyclestage_salesqualifiedlead_date": null, "properties_hs_lifecyclestage_subscriber_date": null, "properties_hs_linkedin_ad_clicked": null, "properties_hs_marketable_reason_id": null, "properties_hs_marketable_reason_type": null, "properties_hs_marketable_status": "false", "properties_hs_marketable_until_renewal": "false", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 401, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_persona": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "contacts-lifecycle-pipeline", "properties_hs_predictivecontactscore": null, "properties_hs_predictivecontactscore_v2": 0.29, "properties_hs_predictivecontactscorebucket": null, "properties_hs_predictivescoringtier": "tier_4", "properties_hs_read_only": null, "properties_hs_sa_first_engagement_date": null, "properties_hs_sa_first_engagement_descr": null, "properties_hs_sa_first_engagement_object_type": null, "properties_hs_sales_email_last_clicked": null, "properties_hs_sales_email_last_opened": null, "properties_hs_sales_email_last_replied": null, "properties_hs_searchable_calculated_international_mobile_number": null, "properties_hs_searchable_calculated_international_phone_number": null, "properties_hs_searchable_calculated_mobile_number": null, "properties_hs_searchable_calculated_phone_number": "8884827768", "properties_hs_sequences_actively_enrolled_count": null, "properties_hs_sequences_enrolled_count": null, "properties_hs_sequences_is_enrolled": null, "properties_hs_testpurge": null, "properties_hs_testrollback": null, "properties_hs_time_between_contact_creation_and_deal_close": null, "properties_hs_time_between_contact_creation_and_deal_creation": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": 87712061483, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": null, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": null, "properties_hs_time_to_first_engagement": null, "properties_hs_time_to_move_from_lead_to_customer": null, "properties_hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_opportunity_to_customer": null, "properties_hs_time_to_move_from_salesqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_subscriber_to_customer": null, "properties_hs_timezone": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_v2_cumulative_time_in_customer": null, "properties_hs_v2_cumulative_time_in_evangelist": null, "properties_hs_v2_cumulative_time_in_lead": null, "properties_hs_v2_cumulative_time_in_marketingqualifiedlead": null, "properties_hs_v2_cumulative_time_in_opportunity": null, "properties_hs_v2_cumulative_time_in_other": null, "properties_hs_v2_cumulative_time_in_salesqualifiedlead": null, "properties_hs_v2_cumulative_time_in_subscriber": null, "properties_hs_v2_date_entered_customer": null, "properties_hs_v2_date_entered_evangelist": null, "properties_hs_v2_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "properties_hs_v2_date_entered_marketingqualifiedlead": null, "properties_hs_v2_date_entered_opportunity": null, "properties_hs_v2_date_entered_other": null, "properties_hs_v2_date_entered_salesqualifiedlead": null, "properties_hs_v2_date_entered_subscriber": null, "properties_hs_v2_date_exited_customer": null, "properties_hs_v2_date_exited_evangelist": null, "properties_hs_v2_date_exited_lead": null, "properties_hs_v2_date_exited_marketingqualifiedlead": null, "properties_hs_v2_date_exited_opportunity": null, "properties_hs_v2_date_exited_other": null, "properties_hs_v2_date_exited_salesqualifiedlead": null, "properties_hs_v2_date_exited_subscriber": null, "properties_hs_v2_latest_time_in_customer": null, "properties_hs_v2_latest_time_in_evangelist": null, "properties_hs_v2_latest_time_in_lead": null, "properties_hs_v2_latest_time_in_marketingqualifiedlead": null, "properties_hs_v2_latest_time_in_opportunity": null, "properties_hs_v2_latest_time_in_other": null, "properties_hs_v2_latest_time_in_salesqualifiedlead": null, "properties_hs_v2_latest_time_in_subscriber": null, "properties_hs_was_imported": true, "properties_hs_whatsapp_phone_number": null, "properties_hubspot_owner_assigneddate": "2021-05-21T10:20:30.963000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_ip_city": null, "properties_ip_country": null, "properties_ip_country_code": null, "properties_ip_latlon": null, "properties_ip_state": null, "properties_ip_state_code": null, "properties_ip_zipcode": null, "properties_job_function": null, "properties_jobtitle": null, "properties_lastmodifieddate": "2023-03-21T19:31:00.563000+00:00", "properties_lastname": "Mitchell", "properties_lifecyclestage": "lead", "properties_marital_status": null, "properties_message": null, "properties_military_status": null, "properties_mobilephone": null, "properties_my_custom_test_property": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_deals": null, "properties_num_contacted_notes": null, "properties_num_conversion_events": 0, "properties_num_notes": null, "properties_num_unique_conversion_events": 0, "properties_numemployees": null, "properties_phone": "1(888) 482-7768", "properties_recent_conversion_date": null, "properties_recent_conversion_event_name": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_relationship_status": null, "properties_salutation": null, "properties_school": null, "properties_seniority": null, "properties_start_date": null, "properties_state": "MA", "properties_surveymonkeyeventlastupdated": null, "properties_test": null, "properties_total_revenue": null, "properties_twitterhandle": null, "properties_webinareventlastupdated": null, "properties_website": null, "properties_work_email": null, "properties_zip": "21430"}, "emitted_at": 1701823098432} +{"stream": "contacts", "data": {"id": "151", "properties": {"address": null, "annualrevenue": null, "associatedcompanyid": 5000526215, "associatedcompanylastupdated": null, "city": null, "closedate": null, "company": null, "company_size": null, "country": null, "createdate": "2020-12-11T01:29:50.116000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "shef@dne.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "she", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "151", "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_2": "CRM_UI", "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_follow_up_enqueued_at": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": 1, "hs_count_is_worked": 0, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "dne.io", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_disqualified_lead_date": null, "hs_latest_meeting_activity": null, "hs_latest_open_lead_date": null, "hs_latest_qualified_lead_date": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "CONTACTS", "hs_latest_source_data_2": "CRM_UI", "hs_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": null, "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": "2020-12-11T01:29:50.116000+00:00", "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 151, "hs_object_source": "CONTACTS", "hs_object_source_id": "CRM_UI", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.3, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_3", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": null, "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": 98536670035, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_v2_cumulative_time_in_customer": null, "hs_v2_cumulative_time_in_evangelist": null, "hs_v2_cumulative_time_in_lead": null, "hs_v2_cumulative_time_in_marketingqualifiedlead": null, "hs_v2_cumulative_time_in_opportunity": null, "hs_v2_cumulative_time_in_other": null, "hs_v2_cumulative_time_in_salesqualifiedlead": null, "hs_v2_cumulative_time_in_subscriber": null, "hs_v2_date_entered_customer": null, "hs_v2_date_entered_evangelist": null, "hs_v2_date_entered_lead": null, "hs_v2_date_entered_marketingqualifiedlead": null, "hs_v2_date_entered_opportunity": null, "hs_v2_date_entered_other": null, "hs_v2_date_entered_salesqualifiedlead": null, "hs_v2_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "hs_v2_date_exited_customer": null, "hs_v2_date_exited_evangelist": null, "hs_v2_date_exited_lead": null, "hs_v2_date_exited_marketingqualifiedlead": null, "hs_v2_date_exited_opportunity": null, "hs_v2_date_exited_other": null, "hs_v2_date_exited_salesqualifiedlead": null, "hs_v2_date_exited_subscriber": null, "hs_v2_latest_time_in_customer": null, "hs_v2_latest_time_in_evangelist": null, "hs_v2_latest_time_in_lead": null, "hs_v2_latest_time_in_marketingqualifiedlead": null, "hs_v2_latest_time_in_opportunity": null, "hs_v2_latest_time_in_other": null, "hs_v2_latest_time_in_salesqualifiedlead": null, "hs_v2_latest_time_in_subscriber": null, "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": "2020-12-11T01:29:50.093000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-11-22T21:10:04.346000+00:00", "lastname": "nad", "lifecyclestage": "subscriber", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": null, "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": null, "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": null, "work_email": null, "zip": null}, "createdAt": "2020-12-11T01:29:50.116Z", "updatedAt": "2023-11-22T21:10:04.346Z", "archived": false, "companies": ["5000526215", "5000526215"], "properties_address": null, "properties_annualrevenue": null, "properties_associatedcompanyid": 5000526215, "properties_associatedcompanylastupdated": null, "properties_city": null, "properties_closedate": null, "properties_company": null, "properties_company_size": null, "properties_country": null, "properties_createdate": "2020-12-11T01:29:50.116000+00:00", "properties_currentlyinworkflow": null, "properties_date_of_birth": null, "properties_days_to_close": null, "properties_degree": null, "properties_email": "shef@dne.io", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_fax": null, "properties_field_of_study": null, "properties_first_conversion_date": null, "properties_first_conversion_event_name": null, "properties_first_deal_created_date": null, "properties_firstname": "she", "properties_gender": null, "properties_graduation_date": null, "properties_hs_additional_emails": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_contact_vids": "151", "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_average_page_views": 0, "properties_hs_analytics_first_referrer": null, "properties_hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_url": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_last_referrer": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_url": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_num_event_completions": 0, "properties_hs_analytics_num_page_views": 0, "properties_hs_analytics_num_visits": 0, "properties_hs_analytics_revenue": 0.0, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "CONTACTS", "properties_hs_analytics_source_data_2": "CRM_UI", "properties_hs_avatar_filemanager_key": null, "properties_hs_buying_role": null, "properties_hs_calculated_form_submissions": null, "properties_hs_calculated_merged_vids": null, "properties_hs_calculated_mobile_number": null, "properties_hs_calculated_phone_number": null, "properties_hs_calculated_phone_number_area_code": null, "properties_hs_calculated_phone_number_country_code": null, "properties_hs_calculated_phone_number_region_code": null, "properties_hs_clicked_linkedin_ad": null, "properties_hs_content_membership_email": null, "properties_hs_content_membership_email_confirmed": null, "properties_hs_content_membership_follow_up_enqueued_at": null, "properties_hs_content_membership_notes": null, "properties_hs_content_membership_registered_at": null, "properties_hs_content_membership_registration_domain_sent_to": null, "properties_hs_content_membership_registration_email_sent_at": null, "properties_hs_content_membership_status": null, "properties_hs_conversations_visitor_email": null, "properties_hs_count_is_unworked": 1, "properties_hs_count_is_worked": 0, "properties_hs_created_by_conversations": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": null, "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_document_last_revisited": null, "properties_hs_email_bad_address": null, "properties_hs_email_bounce": null, "properties_hs_email_click": null, "properties_hs_email_customer_quarantined_reason": null, "properties_hs_email_delivered": null, "properties_hs_email_domain": "dne.io", "properties_hs_email_first_click_date": null, "properties_hs_email_first_open_date": null, "properties_hs_email_first_reply_date": null, "properties_hs_email_first_send_date": null, "properties_hs_email_hard_bounce_reason": null, "properties_hs_email_hard_bounce_reason_enum": null, "properties_hs_email_is_ineligible": null, "properties_hs_email_last_click_date": null, "properties_hs_email_last_email_name": null, "properties_hs_email_last_open_date": null, "properties_hs_email_last_reply_date": null, "properties_hs_email_last_send_date": null, "properties_hs_email_open": null, "properties_hs_email_optout": null, "properties_hs_email_optout_10798197": null, "properties_hs_email_optout_11890603": null, "properties_hs_email_optout_11890831": null, "properties_hs_email_optout_23704464": null, "properties_hs_email_optout_94692364": null, "properties_hs_email_quarantined": null, "properties_hs_email_quarantined_reason": null, "properties_hs_email_recipient_fatigue_recovery_time": null, "properties_hs_email_replied": null, "properties_hs_email_sends_since_last_engagement": null, "properties_hs_emailconfirmationstatus": null, "properties_hs_facebook_ad_clicked": null, "properties_hs_facebook_click_id": null, "properties_hs_feedback_last_nps_follow_up": null, "properties_hs_feedback_last_nps_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_feedback_show_nps_web_survey": null, "properties_hs_first_engagement_object_id": null, "properties_hs_first_outreach_date": null, "properties_hs_first_subscription_create_date": null, "properties_hs_google_click_id": null, "properties_hs_has_active_subscription": null, "properties_hs_ip_timezone": null, "properties_hs_is_contact": true, "properties_hs_is_unworked": true, "properties_hs_language": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": null, "properties_hs_latest_disqualified_lead_date": null, "properties_hs_latest_meeting_activity": null, "properties_hs_latest_open_lead_date": null, "properties_hs_latest_qualified_lead_date": null, "properties_hs_latest_sequence_ended_date": null, "properties_hs_latest_sequence_enrolled": null, "properties_hs_latest_sequence_enrolled_date": null, "properties_hs_latest_sequence_finished_date": null, "properties_hs_latest_sequence_unenrolled_date": null, "properties_hs_latest_source": "OFFLINE", "properties_hs_latest_source_data_1": "CONTACTS", "properties_hs_latest_source_data_2": "CRM_UI", "properties_hs_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "properties_hs_latest_subscription_create_date": null, "properties_hs_lead_status": null, "properties_hs_legal_basis": null, "properties_hs_lifecyclestage_customer_date": null, "properties_hs_lifecyclestage_evangelist_date": null, "properties_hs_lifecyclestage_lead_date": null, "properties_hs_lifecyclestage_marketingqualifiedlead_date": null, "properties_hs_lifecyclestage_opportunity_date": null, "properties_hs_lifecyclestage_other_date": null, "properties_hs_lifecyclestage_salesqualifiedlead_date": null, "properties_hs_lifecyclestage_subscriber_date": "2020-12-11T01:29:50.116000+00:00", "properties_hs_linkedin_ad_clicked": null, "properties_hs_marketable_reason_id": null, "properties_hs_marketable_reason_type": null, "properties_hs_marketable_status": "false", "properties_hs_marketable_until_renewal": "false", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 151, "properties_hs_object_source": "CONTACTS", "properties_hs_object_source_id": "CRM_UI", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_persona": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "contacts-lifecycle-pipeline", "properties_hs_predictivecontactscore": null, "properties_hs_predictivecontactscore_v2": 0.3, "properties_hs_predictivecontactscorebucket": null, "properties_hs_predictivescoringtier": "tier_3", "properties_hs_read_only": null, "properties_hs_sa_first_engagement_date": null, "properties_hs_sa_first_engagement_descr": null, "properties_hs_sa_first_engagement_object_type": null, "properties_hs_sales_email_last_clicked": null, "properties_hs_sales_email_last_opened": null, "properties_hs_sales_email_last_replied": null, "properties_hs_searchable_calculated_international_mobile_number": null, "properties_hs_searchable_calculated_international_phone_number": null, "properties_hs_searchable_calculated_mobile_number": null, "properties_hs_searchable_calculated_phone_number": null, "properties_hs_sequences_actively_enrolled_count": null, "properties_hs_sequences_enrolled_count": null, "properties_hs_sequences_is_enrolled": null, "properties_hs_testpurge": null, "properties_hs_testrollback": null, "properties_hs_time_between_contact_creation_and_deal_close": null, "properties_hs_time_between_contact_creation_and_deal_creation": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": null, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": 98536670035, "properties_hs_time_to_first_engagement": null, "properties_hs_time_to_move_from_lead_to_customer": null, "properties_hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_opportunity_to_customer": null, "properties_hs_time_to_move_from_salesqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_subscriber_to_customer": null, "properties_hs_timezone": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_v2_cumulative_time_in_customer": null, "properties_hs_v2_cumulative_time_in_evangelist": null, "properties_hs_v2_cumulative_time_in_lead": null, "properties_hs_v2_cumulative_time_in_marketingqualifiedlead": null, "properties_hs_v2_cumulative_time_in_opportunity": null, "properties_hs_v2_cumulative_time_in_other": null, "properties_hs_v2_cumulative_time_in_salesqualifiedlead": null, "properties_hs_v2_cumulative_time_in_subscriber": null, "properties_hs_v2_date_entered_customer": null, "properties_hs_v2_date_entered_evangelist": null, "properties_hs_v2_date_entered_lead": null, "properties_hs_v2_date_entered_marketingqualifiedlead": null, "properties_hs_v2_date_entered_opportunity": null, "properties_hs_v2_date_entered_other": null, "properties_hs_v2_date_entered_salesqualifiedlead": null, "properties_hs_v2_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "properties_hs_v2_date_exited_customer": null, "properties_hs_v2_date_exited_evangelist": null, "properties_hs_v2_date_exited_lead": null, "properties_hs_v2_date_exited_marketingqualifiedlead": null, "properties_hs_v2_date_exited_opportunity": null, "properties_hs_v2_date_exited_other": null, "properties_hs_v2_date_exited_salesqualifiedlead": null, "properties_hs_v2_date_exited_subscriber": null, "properties_hs_v2_latest_time_in_customer": null, "properties_hs_v2_latest_time_in_evangelist": null, "properties_hs_v2_latest_time_in_lead": null, "properties_hs_v2_latest_time_in_marketingqualifiedlead": null, "properties_hs_v2_latest_time_in_opportunity": null, "properties_hs_v2_latest_time_in_other": null, "properties_hs_v2_latest_time_in_salesqualifiedlead": null, "properties_hs_v2_latest_time_in_subscriber": null, "properties_hs_was_imported": null, "properties_hs_whatsapp_phone_number": null, "properties_hubspot_owner_assigneddate": "2020-12-11T01:29:50.093000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_ip_city": null, "properties_ip_country": null, "properties_ip_country_code": null, "properties_ip_latlon": null, "properties_ip_state": null, "properties_ip_state_code": null, "properties_ip_zipcode": null, "properties_job_function": null, "properties_jobtitle": null, "properties_lastmodifieddate": "2023-11-22T21:10:04.346000+00:00", "properties_lastname": "nad", "properties_lifecyclestage": "subscriber", "properties_marital_status": null, "properties_message": null, "properties_military_status": null, "properties_mobilephone": null, "properties_my_custom_test_property": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_deals": null, "properties_num_contacted_notes": null, "properties_num_conversion_events": 0, "properties_num_notes": null, "properties_num_unique_conversion_events": 0, "properties_numemployees": null, "properties_phone": null, "properties_recent_conversion_date": null, "properties_recent_conversion_event_name": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_relationship_status": null, "properties_salutation": null, "properties_school": null, "properties_seniority": null, "properties_start_date": null, "properties_state": null, "properties_surveymonkeyeventlastupdated": null, "properties_test": null, "properties_total_revenue": null, "properties_twitterhandle": null, "properties_webinareventlastupdated": null, "properties_website": null, "properties_work_email": null, "properties_zip": null}, "emitted_at": 1706186860294} +{"stream": "contacts", "data": {"id": "251", "properties": {"address": "25000000 First Street", "annualrevenue": null, "associatedcompanyid": 5170561229, "associatedcompanylastupdated": null, "city": "Cambridge", "closedate": null, "company": "HubSpot", "company_size": null, "country": "USA", "createdate": "2021-02-22T14:05:09.944000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "testingdsapis@hubspot.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "Test User 5001", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "251", "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2021-02-22T14:05:09.944000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "API", "hs_analytics_source_data_2": null, "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_follow_up_enqueued_at": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": null, "hs_count_is_worked": null, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "hubspot.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_disqualified_lead_date": null, "hs_latest_meeting_activity": null, "hs_latest_open_lead_date": null, "hs_latest_qualified_lead_date": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "API", "hs_latest_source_data_2": null, "hs_latest_source_timestamp": "2021-02-22T14:05:10.036000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": null, "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": "2021-02-22T14:05:09.944000+00:00", "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 251, "hs_object_source": "API", "hs_object_source_id": null, "hs_object_source_label": "INTERNAL_PROCESSING", "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.29, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_4", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "5551222323", "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": 92184150208, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_v2_cumulative_time_in_customer": null, "hs_v2_cumulative_time_in_evangelist": null, "hs_v2_cumulative_time_in_lead": null, "hs_v2_cumulative_time_in_marketingqualifiedlead": null, "hs_v2_cumulative_time_in_opportunity": null, "hs_v2_cumulative_time_in_other": null, "hs_v2_cumulative_time_in_salesqualifiedlead": null, "hs_v2_cumulative_time_in_subscriber": null, "hs_v2_date_entered_customer": null, "hs_v2_date_entered_evangelist": null, "hs_v2_date_entered_lead": null, "hs_v2_date_entered_marketingqualifiedlead": null, "hs_v2_date_entered_opportunity": null, "hs_v2_date_entered_other": null, "hs_v2_date_entered_salesqualifiedlead": null, "hs_v2_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "hs_v2_date_exited_customer": null, "hs_v2_date_exited_evangelist": null, "hs_v2_date_exited_lead": null, "hs_v2_date_exited_marketingqualifiedlead": null, "hs_v2_date_exited_opportunity": null, "hs_v2_date_exited_other": null, "hs_v2_date_exited_salesqualifiedlead": null, "hs_v2_date_exited_subscriber": null, "hs_v2_latest_time_in_customer": null, "hs_v2_latest_time_in_evangelist": null, "hs_v2_latest_time_in_lead": null, "hs_v2_latest_time_in_marketingqualifiedlead": null, "hs_v2_latest_time_in_opportunity": null, "hs_v2_latest_time_in_other": null, "hs_v2_latest_time_in_salesqualifiedlead": null, "hs_v2_latest_time_in_subscriber": null, "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-03-21T19:29:13.036000+00:00", "lastname": "Test Lastname 5001", "lifecyclestage": "subscriber", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "555-122-2323", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": "MA", "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": "http://hubspot.com", "work_email": null, "zip": "02139"}, "createdAt": "2021-02-22T14:05:09.944Z", "updatedAt": "2023-03-21T19:29:13.036Z", "archived": false, "companies": ["5170561229", "5170561229"], "properties_address": "25000000 First Street", "properties_annualrevenue": null, "properties_associatedcompanyid": 5170561229, "properties_associatedcompanylastupdated": null, "properties_city": "Cambridge", "properties_closedate": null, "properties_company": "HubSpot", "properties_company_size": null, "properties_country": "USA", "properties_createdate": "2021-02-22T14:05:09.944000+00:00", "properties_currentlyinworkflow": null, "properties_date_of_birth": null, "properties_days_to_close": null, "properties_degree": null, "properties_email": "testingdsapis@hubspot.com", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_fax": null, "properties_field_of_study": null, "properties_first_conversion_date": null, "properties_first_conversion_event_name": null, "properties_first_deal_created_date": null, "properties_firstname": "Test User 5001", "properties_gender": null, "properties_graduation_date": null, "properties_hs_additional_emails": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_contact_vids": "251", "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_analytics_average_page_views": 0, "properties_hs_analytics_first_referrer": null, "properties_hs_analytics_first_timestamp": "2021-02-22T14:05:09.944000+00:00", "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_url": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_last_referrer": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_url": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_num_event_completions": 0, "properties_hs_analytics_num_page_views": 0, "properties_hs_analytics_num_visits": 0, "properties_hs_analytics_revenue": 0.0, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "API", "properties_hs_analytics_source_data_2": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_buying_role": null, "properties_hs_calculated_form_submissions": null, "properties_hs_calculated_merged_vids": null, "properties_hs_calculated_mobile_number": null, "properties_hs_calculated_phone_number": null, "properties_hs_calculated_phone_number_area_code": null, "properties_hs_calculated_phone_number_country_code": null, "properties_hs_calculated_phone_number_region_code": null, "properties_hs_clicked_linkedin_ad": null, "properties_hs_content_membership_email": null, "properties_hs_content_membership_email_confirmed": null, "properties_hs_content_membership_follow_up_enqueued_at": null, "properties_hs_content_membership_notes": null, "properties_hs_content_membership_registered_at": null, "properties_hs_content_membership_registration_domain_sent_to": null, "properties_hs_content_membership_registration_email_sent_at": null, "properties_hs_content_membership_status": null, "properties_hs_conversations_visitor_email": null, "properties_hs_count_is_unworked": null, "properties_hs_count_is_worked": null, "properties_hs_created_by_conversations": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": null, "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_document_last_revisited": null, "properties_hs_email_bad_address": null, "properties_hs_email_bounce": null, "properties_hs_email_click": null, "properties_hs_email_customer_quarantined_reason": null, "properties_hs_email_delivered": null, "properties_hs_email_domain": "hubspot.com", "properties_hs_email_first_click_date": null, "properties_hs_email_first_open_date": null, "properties_hs_email_first_reply_date": null, "properties_hs_email_first_send_date": null, "properties_hs_email_hard_bounce_reason": null, "properties_hs_email_hard_bounce_reason_enum": null, "properties_hs_email_is_ineligible": null, "properties_hs_email_last_click_date": null, "properties_hs_email_last_email_name": null, "properties_hs_email_last_open_date": null, "properties_hs_email_last_reply_date": null, "properties_hs_email_last_send_date": null, "properties_hs_email_open": null, "properties_hs_email_optout": null, "properties_hs_email_optout_10798197": null, "properties_hs_email_optout_11890603": null, "properties_hs_email_optout_11890831": null, "properties_hs_email_optout_23704464": null, "properties_hs_email_optout_94692364": null, "properties_hs_email_quarantined": null, "properties_hs_email_quarantined_reason": null, "properties_hs_email_recipient_fatigue_recovery_time": null, "properties_hs_email_replied": null, "properties_hs_email_sends_since_last_engagement": null, "properties_hs_emailconfirmationstatus": null, "properties_hs_facebook_ad_clicked": null, "properties_hs_facebook_click_id": null, "properties_hs_feedback_last_nps_follow_up": null, "properties_hs_feedback_last_nps_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_feedback_show_nps_web_survey": null, "properties_hs_first_engagement_object_id": null, "properties_hs_first_outreach_date": null, "properties_hs_first_subscription_create_date": null, "properties_hs_google_click_id": null, "properties_hs_has_active_subscription": null, "properties_hs_ip_timezone": null, "properties_hs_is_contact": true, "properties_hs_is_unworked": true, "properties_hs_language": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": null, "properties_hs_latest_disqualified_lead_date": null, "properties_hs_latest_meeting_activity": null, "properties_hs_latest_open_lead_date": null, "properties_hs_latest_qualified_lead_date": null, "properties_hs_latest_sequence_ended_date": null, "properties_hs_latest_sequence_enrolled": null, "properties_hs_latest_sequence_enrolled_date": null, "properties_hs_latest_sequence_finished_date": null, "properties_hs_latest_sequence_unenrolled_date": null, "properties_hs_latest_source": "OFFLINE", "properties_hs_latest_source_data_1": "API", "properties_hs_latest_source_data_2": null, "properties_hs_latest_source_timestamp": "2021-02-22T14:05:10.036000+00:00", "properties_hs_latest_subscription_create_date": null, "properties_hs_lead_status": null, "properties_hs_legal_basis": null, "properties_hs_lifecyclestage_customer_date": null, "properties_hs_lifecyclestage_evangelist_date": null, "properties_hs_lifecyclestage_lead_date": null, "properties_hs_lifecyclestage_marketingqualifiedlead_date": null, "properties_hs_lifecyclestage_opportunity_date": null, "properties_hs_lifecyclestage_other_date": null, "properties_hs_lifecyclestage_salesqualifiedlead_date": null, "properties_hs_lifecyclestage_subscriber_date": "2021-02-22T14:05:09.944000+00:00", "properties_hs_linkedin_ad_clicked": null, "properties_hs_marketable_reason_id": null, "properties_hs_marketable_reason_type": null, "properties_hs_marketable_status": "false", "properties_hs_marketable_until_renewal": "false", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 251, "properties_hs_object_source": "API", "properties_hs_object_source_id": null, "properties_hs_object_source_label": "INTERNAL_PROCESSING", "properties_hs_object_source_user_id": null, "properties_hs_persona": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "contacts-lifecycle-pipeline", "properties_hs_predictivecontactscore": null, "properties_hs_predictivecontactscore_v2": 0.29, "properties_hs_predictivecontactscorebucket": null, "properties_hs_predictivescoringtier": "tier_4", "properties_hs_read_only": null, "properties_hs_sa_first_engagement_date": null, "properties_hs_sa_first_engagement_descr": null, "properties_hs_sa_first_engagement_object_type": null, "properties_hs_sales_email_last_clicked": null, "properties_hs_sales_email_last_opened": null, "properties_hs_sales_email_last_replied": null, "properties_hs_searchable_calculated_international_mobile_number": null, "properties_hs_searchable_calculated_international_phone_number": null, "properties_hs_searchable_calculated_mobile_number": null, "properties_hs_searchable_calculated_phone_number": "5551222323", "properties_hs_sequences_actively_enrolled_count": null, "properties_hs_sequences_enrolled_count": null, "properties_hs_sequences_is_enrolled": null, "properties_hs_testpurge": null, "properties_hs_testrollback": null, "properties_hs_time_between_contact_creation_and_deal_close": null, "properties_hs_time_between_contact_creation_and_deal_creation": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": null, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": 92184150208, "properties_hs_time_to_first_engagement": null, "properties_hs_time_to_move_from_lead_to_customer": null, "properties_hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_opportunity_to_customer": null, "properties_hs_time_to_move_from_salesqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_subscriber_to_customer": null, "properties_hs_timezone": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_v2_cumulative_time_in_customer": null, "properties_hs_v2_cumulative_time_in_evangelist": null, "properties_hs_v2_cumulative_time_in_lead": null, "properties_hs_v2_cumulative_time_in_marketingqualifiedlead": null, "properties_hs_v2_cumulative_time_in_opportunity": null, "properties_hs_v2_cumulative_time_in_other": null, "properties_hs_v2_cumulative_time_in_salesqualifiedlead": null, "properties_hs_v2_cumulative_time_in_subscriber": null, "properties_hs_v2_date_entered_customer": null, "properties_hs_v2_date_entered_evangelist": null, "properties_hs_v2_date_entered_lead": null, "properties_hs_v2_date_entered_marketingqualifiedlead": null, "properties_hs_v2_date_entered_opportunity": null, "properties_hs_v2_date_entered_other": null, "properties_hs_v2_date_entered_salesqualifiedlead": null, "properties_hs_v2_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "properties_hs_v2_date_exited_customer": null, "properties_hs_v2_date_exited_evangelist": null, "properties_hs_v2_date_exited_lead": null, "properties_hs_v2_date_exited_marketingqualifiedlead": null, "properties_hs_v2_date_exited_opportunity": null, "properties_hs_v2_date_exited_other": null, "properties_hs_v2_date_exited_salesqualifiedlead": null, "properties_hs_v2_date_exited_subscriber": null, "properties_hs_v2_latest_time_in_customer": null, "properties_hs_v2_latest_time_in_evangelist": null, "properties_hs_v2_latest_time_in_lead": null, "properties_hs_v2_latest_time_in_marketingqualifiedlead": null, "properties_hs_v2_latest_time_in_opportunity": null, "properties_hs_v2_latest_time_in_other": null, "properties_hs_v2_latest_time_in_salesqualifiedlead": null, "properties_hs_v2_latest_time_in_subscriber": null, "properties_hs_was_imported": null, "properties_hs_whatsapp_phone_number": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_ip_city": null, "properties_ip_country": null, "properties_ip_country_code": null, "properties_ip_latlon": null, "properties_ip_state": null, "properties_ip_state_code": null, "properties_ip_zipcode": null, "properties_job_function": null, "properties_jobtitle": null, "properties_lastmodifieddate": "2023-03-21T19:29:13.036000+00:00", "properties_lastname": "Test Lastname 5001", "properties_lifecyclestage": "subscriber", "properties_marital_status": null, "properties_message": null, "properties_military_status": null, "properties_mobilephone": null, "properties_my_custom_test_property": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_deals": null, "properties_num_contacted_notes": null, "properties_num_conversion_events": 0, "properties_num_notes": null, "properties_num_unique_conversion_events": 0, "properties_numemployees": null, "properties_phone": "555-122-2323", "properties_recent_conversion_date": null, "properties_recent_conversion_event_name": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_relationship_status": null, "properties_salutation": null, "properties_school": null, "properties_seniority": null, "properties_start_date": null, "properties_state": "MA", "properties_surveymonkeyeventlastupdated": null, "properties_test": null, "properties_total_revenue": null, "properties_twitterhandle": null, "properties_webinareventlastupdated": null, "properties_website": "http://hubspot.com", "properties_work_email": null, "properties_zip": "02139"}, "emitted_at": 1706186860299} +{"stream": "contacts", "data": {"id": "401", "properties": {"address": "25 First Street", "annualrevenue": null, "associatedcompanyid": null, "associatedcompanylastupdated": null, "city": "Cambridge", "closedate": null, "company": null, "company_size": null, "country": null, "createdate": "2021-02-23T20:10:36.191000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "macmitch@hubspot.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "Mac", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "401", "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2021-02-23T20:10:36.181000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "IMPORT", "hs_analytics_source_data_2": "13256565", "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": "+18884827768", "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": "US", "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_follow_up_enqueued_at": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": 1, "hs_count_is_worked": 0, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "hubspot.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": "OTHER", "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_disqualified_lead_date": null, "hs_latest_meeting_activity": null, "hs_latest_open_lead_date": null, "hs_latest_qualified_lead_date": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "IMPORT", "hs_latest_source_data_2": "13256565", "hs_latest_source_timestamp": "2021-02-23T20:10:36.210000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": "2021-02-23T20:10:36.181000+00:00", "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": null, "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 401, "hs_object_source": "IMPORT", "hs_object_source_id": "13256565", "hs_object_source_label": "IMPORT", "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.29, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_4", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "8884827768", "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": 92075823971, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_v2_cumulative_time_in_customer": null, "hs_v2_cumulative_time_in_evangelist": null, "hs_v2_cumulative_time_in_lead": null, "hs_v2_cumulative_time_in_marketingqualifiedlead": null, "hs_v2_cumulative_time_in_opportunity": null, "hs_v2_cumulative_time_in_other": null, "hs_v2_cumulative_time_in_salesqualifiedlead": null, "hs_v2_cumulative_time_in_subscriber": null, "hs_v2_date_entered_customer": null, "hs_v2_date_entered_evangelist": null, "hs_v2_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "hs_v2_date_entered_marketingqualifiedlead": null, "hs_v2_date_entered_opportunity": null, "hs_v2_date_entered_other": null, "hs_v2_date_entered_salesqualifiedlead": null, "hs_v2_date_entered_subscriber": null, "hs_v2_date_exited_customer": null, "hs_v2_date_exited_evangelist": null, "hs_v2_date_exited_lead": null, "hs_v2_date_exited_marketingqualifiedlead": null, "hs_v2_date_exited_opportunity": null, "hs_v2_date_exited_other": null, "hs_v2_date_exited_salesqualifiedlead": null, "hs_v2_date_exited_subscriber": null, "hs_v2_latest_time_in_customer": null, "hs_v2_latest_time_in_evangelist": null, "hs_v2_latest_time_in_lead": null, "hs_v2_latest_time_in_marketingqualifiedlead": null, "hs_v2_latest_time_in_opportunity": null, "hs_v2_latest_time_in_other": null, "hs_v2_latest_time_in_salesqualifiedlead": null, "hs_v2_latest_time_in_subscriber": null, "hs_was_imported": true, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": "2021-05-21T10:20:30.963000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-03-21T19:31:00.563000+00:00", "lastname": "Mitchell", "lifecyclestage": "lead", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "1(888) 482-7768", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": "MA", "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": null, "work_email": null, "zip": "21430"}, "createdAt": "2021-02-23T20:10:36.191Z", "updatedAt": "2023-03-21T19:31:00.563Z", "archived": false, "properties_address": "25 First Street", "properties_annualrevenue": null, "properties_associatedcompanyid": null, "properties_associatedcompanylastupdated": null, "properties_city": "Cambridge", "properties_closedate": null, "properties_company": null, "properties_company_size": null, "properties_country": null, "properties_createdate": "2021-02-23T20:10:36.191000+00:00", "properties_currentlyinworkflow": null, "properties_date_of_birth": null, "properties_days_to_close": null, "properties_degree": null, "properties_email": "macmitch@hubspot.com", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_fax": null, "properties_field_of_study": null, "properties_first_conversion_date": null, "properties_first_conversion_event_name": null, "properties_first_deal_created_date": null, "properties_firstname": "Mac", "properties_gender": null, "properties_graduation_date": null, "properties_hs_additional_emails": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_contact_vids": "401", "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_average_page_views": 0, "properties_hs_analytics_first_referrer": null, "properties_hs_analytics_first_timestamp": "2021-02-23T20:10:36.181000+00:00", "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_url": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_last_referrer": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_url": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_num_event_completions": 0, "properties_hs_analytics_num_page_views": 0, "properties_hs_analytics_num_visits": 0, "properties_hs_analytics_revenue": 0.0, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "IMPORT", "properties_hs_analytics_source_data_2": "13256565", "properties_hs_avatar_filemanager_key": null, "properties_hs_buying_role": null, "properties_hs_calculated_form_submissions": null, "properties_hs_calculated_merged_vids": null, "properties_hs_calculated_mobile_number": null, "properties_hs_calculated_phone_number": "+18884827768", "properties_hs_calculated_phone_number_area_code": null, "properties_hs_calculated_phone_number_country_code": "US", "properties_hs_calculated_phone_number_region_code": null, "properties_hs_clicked_linkedin_ad": null, "properties_hs_content_membership_email": null, "properties_hs_content_membership_email_confirmed": null, "properties_hs_content_membership_follow_up_enqueued_at": null, "properties_hs_content_membership_notes": null, "properties_hs_content_membership_registered_at": null, "properties_hs_content_membership_registration_domain_sent_to": null, "properties_hs_content_membership_registration_email_sent_at": null, "properties_hs_content_membership_status": null, "properties_hs_conversations_visitor_email": null, "properties_hs_count_is_unworked": 1, "properties_hs_count_is_worked": 0, "properties_hs_created_by_conversations": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": null, "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": null, "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_document_last_revisited": null, "properties_hs_email_bad_address": null, "properties_hs_email_bounce": null, "properties_hs_email_click": null, "properties_hs_email_customer_quarantined_reason": null, "properties_hs_email_delivered": null, "properties_hs_email_domain": "hubspot.com", "properties_hs_email_first_click_date": null, "properties_hs_email_first_open_date": null, "properties_hs_email_first_reply_date": null, "properties_hs_email_first_send_date": null, "properties_hs_email_hard_bounce_reason": null, "properties_hs_email_hard_bounce_reason_enum": "OTHER", "properties_hs_email_is_ineligible": null, "properties_hs_email_last_click_date": null, "properties_hs_email_last_email_name": null, "properties_hs_email_last_open_date": null, "properties_hs_email_last_reply_date": null, "properties_hs_email_last_send_date": null, "properties_hs_email_open": null, "properties_hs_email_optout": null, "properties_hs_email_optout_10798197": null, "properties_hs_email_optout_11890603": null, "properties_hs_email_optout_11890831": null, "properties_hs_email_optout_23704464": null, "properties_hs_email_optout_94692364": null, "properties_hs_email_quarantined": null, "properties_hs_email_quarantined_reason": null, "properties_hs_email_recipient_fatigue_recovery_time": null, "properties_hs_email_replied": null, "properties_hs_email_sends_since_last_engagement": null, "properties_hs_emailconfirmationstatus": null, "properties_hs_facebook_ad_clicked": null, "properties_hs_facebook_click_id": null, "properties_hs_feedback_last_nps_follow_up": null, "properties_hs_feedback_last_nps_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_feedback_show_nps_web_survey": null, "properties_hs_first_engagement_object_id": null, "properties_hs_first_outreach_date": null, "properties_hs_first_subscription_create_date": null, "properties_hs_google_click_id": null, "properties_hs_has_active_subscription": null, "properties_hs_ip_timezone": null, "properties_hs_is_contact": true, "properties_hs_is_unworked": true, "properties_hs_language": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": null, "properties_hs_latest_disqualified_lead_date": null, "properties_hs_latest_meeting_activity": null, "properties_hs_latest_open_lead_date": null, "properties_hs_latest_qualified_lead_date": null, "properties_hs_latest_sequence_ended_date": null, "properties_hs_latest_sequence_enrolled": null, "properties_hs_latest_sequence_enrolled_date": null, "properties_hs_latest_sequence_finished_date": null, "properties_hs_latest_sequence_unenrolled_date": null, "properties_hs_latest_source": "OFFLINE", "properties_hs_latest_source_data_1": "IMPORT", "properties_hs_latest_source_data_2": "13256565", "properties_hs_latest_source_timestamp": "2021-02-23T20:10:36.210000+00:00", "properties_hs_latest_subscription_create_date": null, "properties_hs_lead_status": null, "properties_hs_legal_basis": null, "properties_hs_lifecyclestage_customer_date": null, "properties_hs_lifecyclestage_evangelist_date": null, "properties_hs_lifecyclestage_lead_date": "2021-02-23T20:10:36.181000+00:00", "properties_hs_lifecyclestage_marketingqualifiedlead_date": null, "properties_hs_lifecyclestage_opportunity_date": null, "properties_hs_lifecyclestage_other_date": null, "properties_hs_lifecyclestage_salesqualifiedlead_date": null, "properties_hs_lifecyclestage_subscriber_date": null, "properties_hs_linkedin_ad_clicked": null, "properties_hs_marketable_reason_id": null, "properties_hs_marketable_reason_type": null, "properties_hs_marketable_status": "false", "properties_hs_marketable_until_renewal": "false", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 401, "properties_hs_object_source": "IMPORT", "properties_hs_object_source_id": "13256565", "properties_hs_object_source_label": "IMPORT", "properties_hs_object_source_user_id": null, "properties_hs_persona": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "contacts-lifecycle-pipeline", "properties_hs_predictivecontactscore": null, "properties_hs_predictivecontactscore_v2": 0.29, "properties_hs_predictivecontactscorebucket": null, "properties_hs_predictivescoringtier": "tier_4", "properties_hs_read_only": null, "properties_hs_sa_first_engagement_date": null, "properties_hs_sa_first_engagement_descr": null, "properties_hs_sa_first_engagement_object_type": null, "properties_hs_sales_email_last_clicked": null, "properties_hs_sales_email_last_opened": null, "properties_hs_sales_email_last_replied": null, "properties_hs_searchable_calculated_international_mobile_number": null, "properties_hs_searchable_calculated_international_phone_number": null, "properties_hs_searchable_calculated_mobile_number": null, "properties_hs_searchable_calculated_phone_number": "8884827768", "properties_hs_sequences_actively_enrolled_count": null, "properties_hs_sequences_enrolled_count": null, "properties_hs_sequences_is_enrolled": null, "properties_hs_testpurge": null, "properties_hs_testrollback": null, "properties_hs_time_between_contact_creation_and_deal_close": null, "properties_hs_time_between_contact_creation_and_deal_creation": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": 92075823971, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": null, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": null, "properties_hs_time_to_first_engagement": null, "properties_hs_time_to_move_from_lead_to_customer": null, "properties_hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_opportunity_to_customer": null, "properties_hs_time_to_move_from_salesqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_subscriber_to_customer": null, "properties_hs_timezone": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_v2_cumulative_time_in_customer": null, "properties_hs_v2_cumulative_time_in_evangelist": null, "properties_hs_v2_cumulative_time_in_lead": null, "properties_hs_v2_cumulative_time_in_marketingqualifiedlead": null, "properties_hs_v2_cumulative_time_in_opportunity": null, "properties_hs_v2_cumulative_time_in_other": null, "properties_hs_v2_cumulative_time_in_salesqualifiedlead": null, "properties_hs_v2_cumulative_time_in_subscriber": null, "properties_hs_v2_date_entered_customer": null, "properties_hs_v2_date_entered_evangelist": null, "properties_hs_v2_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "properties_hs_v2_date_entered_marketingqualifiedlead": null, "properties_hs_v2_date_entered_opportunity": null, "properties_hs_v2_date_entered_other": null, "properties_hs_v2_date_entered_salesqualifiedlead": null, "properties_hs_v2_date_entered_subscriber": null, "properties_hs_v2_date_exited_customer": null, "properties_hs_v2_date_exited_evangelist": null, "properties_hs_v2_date_exited_lead": null, "properties_hs_v2_date_exited_marketingqualifiedlead": null, "properties_hs_v2_date_exited_opportunity": null, "properties_hs_v2_date_exited_other": null, "properties_hs_v2_date_exited_salesqualifiedlead": null, "properties_hs_v2_date_exited_subscriber": null, "properties_hs_v2_latest_time_in_customer": null, "properties_hs_v2_latest_time_in_evangelist": null, "properties_hs_v2_latest_time_in_lead": null, "properties_hs_v2_latest_time_in_marketingqualifiedlead": null, "properties_hs_v2_latest_time_in_opportunity": null, "properties_hs_v2_latest_time_in_other": null, "properties_hs_v2_latest_time_in_salesqualifiedlead": null, "properties_hs_v2_latest_time_in_subscriber": null, "properties_hs_was_imported": true, "properties_hs_whatsapp_phone_number": null, "properties_hubspot_owner_assigneddate": "2021-05-21T10:20:30.963000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_ip_city": null, "properties_ip_country": null, "properties_ip_country_code": null, "properties_ip_latlon": null, "properties_ip_state": null, "properties_ip_state_code": null, "properties_ip_zipcode": null, "properties_job_function": null, "properties_jobtitle": null, "properties_lastmodifieddate": "2023-03-21T19:31:00.563000+00:00", "properties_lastname": "Mitchell", "properties_lifecyclestage": "lead", "properties_marital_status": null, "properties_message": null, "properties_military_status": null, "properties_mobilephone": null, "properties_my_custom_test_property": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_deals": null, "properties_num_contacted_notes": null, "properties_num_conversion_events": 0, "properties_num_notes": null, "properties_num_unique_conversion_events": 0, "properties_numemployees": null, "properties_phone": "1(888) 482-7768", "properties_recent_conversion_date": null, "properties_recent_conversion_event_name": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_relationship_status": null, "properties_salutation": null, "properties_school": null, "properties_seniority": null, "properties_start_date": null, "properties_state": "MA", "properties_surveymonkeyeventlastupdated": null, "properties_test": null, "properties_total_revenue": null, "properties_twitterhandle": null, "properties_webinareventlastupdated": null, "properties_website": null, "properties_work_email": null, "properties_zip": "21430"}, "emitted_at": 1706186860302} {"stream": "contacts_list_memberships", "data": {"canonical-vid": 401, "static-list-id": 60, "internal-list-id": 2147483643, "timestamp": 1614111042672, "vid": 401, "is-member": true}, "emitted_at": 1697714191502} {"stream": "contacts_list_memberships", "data": {"canonical-vid": 401, "static-list-id": 61, "internal-list-id": 2147483643, "timestamp": 1615502112726, "vid": 401, "is-member": true}, "emitted_at": 1697714191513} {"stream": "contacts_list_memberships", "data": {"canonical-vid": 2501, "static-list-id": 60, "internal-list-id": 2147483643, "timestamp": 1675124235515, "vid": 2501, "is-member": true}, "emitted_at": 1697714191513} {"stream": "contacts_merged_audit", "data": {"canonical-vid": 651, "vid-to-merge": 201, "timestamp": 1688758327178, "entity-id": "auth:app-cookie | auth-level:app | login-id:integration-test@airbyte.io-1688758203663 | hub-id:8727216 | user-id:12282590 | origin-ip:2804:1b3:8402:b1f4:7d1b:f62e:b071:593d | correlation-id:3f139cd7-66fc-4300-8cbc-e6c1fe9ea7d1", "user-id": 12282590, "num-properties-moved": 45, "merged_from_email": {"value": "testingapis@hubspot.com", "source-type": "API", "source-id": null, "source-label": null, "updated-by-user-id": null, "timestamp": 1610634377014, "selected": false}, "merged_to_email": {"value": "testingapicontact_1@hubspot.com", "source-type": "API", "source-id": null, "source-label": null, "updated-by-user-id": null, "timestamp": 1634044981830, "selected": false}, "first-name": "test", "last-name": "testerson", "merged_from_email_value": "testingapis@hubspot.com", "merged_from_email_source-type": "API", "merged_from_email_source-id": null, "merged_from_email_source-label": null, "merged_from_email_updated-by-user-id": null, "merged_from_email_timestamp": 1610634377014, "merged_from_email_selected": false, "merged_to_email_value": "testingapicontact_1@hubspot.com", "merged_to_email_source-type": "API", "merged_to_email_source-id": null, "merged_to_email_source-label": null, "merged_to_email_updated-by-user-id": null, "merged_to_email_timestamp": 1634044981830, "merged_to_email_selected": false}, "emitted_at": 1697714194351} {"stream": "deal_pipelines", "data": {"label": "New Business Pipeline", "displayOrder": 3, "active": true, "stages": [{"label": "Initial Qualification", "displayOrder": 0, "metadata": {"isClosed": "false", "probability": "0.1"}, "stageId": "9567448", "createdAt": 1610635973956, "updatedAt": 1680620354263, "active": true}, {"label": "Success! Closed Won", "displayOrder": 2, "metadata": {"isClosed": "true", "probability": "1.0"}, "stageId": "customclosedwonstage", "createdAt": 1610635973956, "updatedAt": 1680620354263, "active": true}, {"label": "Negotiation", "displayOrder": 1, "metadata": {"isClosed": "false", "probability": "0.5"}, "stageId": "9567449", "createdAt": 1610635973956, "updatedAt": 1680620354263, "active": true}, {"label": "Closed Lost", "displayOrder": 3, "metadata": {"isClosed": "false", "probability": "0.1"}, "stageId": "66894120", "createdAt": 1680620354263, "updatedAt": 1680620354263, "active": true}], "objectType": "DEAL", "objectTypeId": "0-3", "pipelineId": "b9152945-a594-4835-9676-a6f405fecd71", "createdAt": 1610635973956, "updatedAt": 1680620354263, "default": false}, "emitted_at": 1697714195524} -{"stream": "deals", "data": {"id": "3980651569", "properties": {"amount": 60000, "amount_in_home_currency": 60000, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2014-08-31T00:00:00+00:00", "createdate": "2021-01-13T10:30:42.221000+00:00", "days_to_close": 0, "dealname": "Tim's Newer Deal", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": null, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": "OFFLINE", "hs_analytics_latest_source_company": "OFFLINE", "hs_analytics_latest_source_contact": null, "hs_analytics_latest_source_data_1": "CONTACTS", "hs_analytics_latest_source_data_1_company": "CONTACTS", "hs_analytics_latest_source_data_1_contact": null, "hs_analytics_latest_source_data_2": "CRM_UI", "hs_analytics_latest_source_data_2_company": "CRM_UI", "hs_analytics_latest_source_data_2_contact": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_2": "CRM_UI", "hs_arr": null, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-13T10:30:42.221000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-01-13T10:30:42.221000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_days_to_close_raw": 0, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": null, "hs_exchange_rate": null, "hs_forecast_amount": 60000, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_is_open_count": 1, "hs_lastmodifieddate": "2021-09-07T02:36:16.363000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": null, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": 0, "hs_num_target_accounts": 0, "hs_object_id": 3980651569, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 12000.0, "hs_projected_amount_in_home_currency": 12000.0, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": null, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 92113120720, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_v2_cumulative_time_in_66894120": null, "hs_v2_cumulative_time_in_9567448": null, "hs_v2_cumulative_time_in_9567449": null, "hs_v2_cumulative_time_in_customclosedwonstage": null, "hs_v2_date_entered_66894120": null, "hs_v2_date_entered_9567448": null, "hs_v2_date_entered_9567449": null, "hs_v2_date_entered_customclosedwonstage": null, "hs_v2_date_exited_66894120": null, "hs_v2_date_exited_9567448": null, "hs_v2_date_exited_9567449": null, "hs_v2_date_exited_customclosedwonstage": null, "hs_v2_latest_time_in_66894120": null, "hs_v2_latest_time_in_9567448": null, "hs_v2_latest_time_in_9567449": null, "hs_v2_latest_time_in_customclosedwonstage": null, "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-01-13T10:30:42.221000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-01-13T10:30:42.221Z", "updatedAt": "2021-09-07T02:36:16.363Z", "archived": false, "companies": ["5000526215", "5000526215"], "properties_amount": 60000, "properties_amount_in_home_currency": 60000, "properties_closed_lost_reason": null, "properties_closed_won_reason": null, "properties_closedate": "2014-08-31T00:00:00+00:00", "properties_createdate": "2021-01-13T10:30:42.221000+00:00", "properties_days_to_close": 0, "properties_dealname": "Tim's Newer Deal", "properties_dealstage": "appointmentscheduled", "properties_dealtype": "newbusiness", "properties_description": null, "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_hs_acv": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_collaborator_owner_ids": null, "properties_hs_all_deal_split_owner_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_latest_source": "OFFLINE", "properties_hs_analytics_latest_source_company": "OFFLINE", "properties_hs_analytics_latest_source_contact": null, "properties_hs_analytics_latest_source_data_1": "CONTACTS", "properties_hs_analytics_latest_source_data_1_company": "CONTACTS", "properties_hs_analytics_latest_source_data_1_contact": null, "properties_hs_analytics_latest_source_data_2": "CRM_UI", "properties_hs_analytics_latest_source_data_2_company": "CRM_UI", "properties_hs_analytics_latest_source_data_2_contact": null, "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_latest_source_timestamp_company": null, "properties_hs_analytics_latest_source_timestamp_contact": null, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "CONTACTS", "properties_hs_analytics_source_data_2": "CRM_UI", "properties_hs_arr": null, "properties_hs_campaign": null, "properties_hs_closed_amount": 0, "properties_hs_closed_amount_in_home_currency": 0, "properties_hs_closed_won_count": null, "properties_hs_closed_won_date": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-13T10:30:42.221000+00:00", "properties_hs_date_entered_66894120": null, "properties_hs_date_entered_9567448": null, "properties_hs_date_entered_9567449": null, "properties_hs_date_entered_appointmentscheduled": "2021-01-13T10:30:42.221000+00:00", "properties_hs_date_entered_closedlost": null, "properties_hs_date_entered_closedwon": null, "properties_hs_date_entered_contractsent": null, "properties_hs_date_entered_customclosedwonstage": null, "properties_hs_date_entered_decisionmakerboughtin": null, "properties_hs_date_entered_presentationscheduled": null, "properties_hs_date_entered_qualifiedtobuy": null, "properties_hs_date_exited_66894120": null, "properties_hs_date_exited_9567448": null, "properties_hs_date_exited_9567449": null, "properties_hs_date_exited_appointmentscheduled": null, "properties_hs_date_exited_closedlost": null, "properties_hs_date_exited_closedwon": null, "properties_hs_date_exited_contractsent": null, "properties_hs_date_exited_customclosedwonstage": null, "properties_hs_date_exited_decisionmakerboughtin": null, "properties_hs_date_exited_presentationscheduled": null, "properties_hs_date_exited_qualifiedtobuy": null, "properties_hs_days_to_close_raw": 0, "properties_hs_deal_amount_calculation_preference": null, "properties_hs_deal_stage_probability": 0.2, "properties_hs_deal_stage_probability_shadow": null, "properties_hs_exchange_rate": null, "properties_hs_forecast_amount": 60000, "properties_hs_forecast_probability": null, "properties_hs_is_closed": false, "properties_hs_is_closed_won": false, "properties_hs_is_deal_split": false, "properties_hs_is_open_count": 1, "properties_hs_lastmodifieddate": "2021-09-07T02:36:16.363000+00:00", "properties_hs_latest_meeting_activity": null, "properties_hs_likelihood_to_close": null, "properties_hs_line_item_global_term_hs_discount_percentage": null, "properties_hs_line_item_global_term_hs_discount_percentage_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_period": null, "properties_hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "properties_hs_line_item_global_term_recurringbillingfrequency": null, "properties_hs_line_item_global_term_recurringbillingfrequency_enabled": null, "properties_hs_manual_forecast_category": null, "properties_hs_merged_object_ids": null, "properties_hs_mrr": null, "properties_hs_next_step": null, "properties_hs_num_associated_deal_splits": null, "properties_hs_num_of_associated_line_items": 0, "properties_hs_num_target_accounts": 0, "properties_hs_object_id": 3980651569, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_predicted_amount": null, "properties_hs_predicted_amount_in_home_currency": null, "properties_hs_priority": null, "properties_hs_projected_amount": 12000.0, "properties_hs_projected_amount_in_home_currency": 12000.0, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_tcv": null, "properties_hs_time_in_66894120": null, "properties_hs_time_in_9567448": null, "properties_hs_time_in_9567449": null, "properties_hs_time_in_appointmentscheduled": 92113120720, "properties_hs_time_in_closedlost": null, "properties_hs_time_in_closedwon": null, "properties_hs_time_in_contractsent": null, "properties_hs_time_in_customclosedwonstage": null, "properties_hs_time_in_decisionmakerboughtin": null, "properties_hs_time_in_presentationscheduled": null, "properties_hs_time_in_qualifiedtobuy": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_v2_cumulative_time_in_66894120": null, "properties_hs_v2_cumulative_time_in_9567448": null, "properties_hs_v2_cumulative_time_in_9567449": null, "properties_hs_v2_cumulative_time_in_customclosedwonstage": null, "properties_hs_v2_date_entered_66894120": null, "properties_hs_v2_date_entered_9567448": null, "properties_hs_v2_date_entered_9567449": null, "properties_hs_v2_date_entered_customclosedwonstage": null, "properties_hs_v2_date_exited_66894120": null, "properties_hs_v2_date_exited_9567448": null, "properties_hs_v2_date_exited_9567449": null, "properties_hs_v2_date_exited_customclosedwonstage": null, "properties_hs_v2_latest_time_in_66894120": null, "properties_hs_v2_latest_time_in_9567448": null, "properties_hs_v2_latest_time_in_9567449": null, "properties_hs_v2_latest_time_in_customclosedwonstage": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2021-01-13T10:30:42.221000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_pipeline": "default"}, "emitted_at": 1702646964420} -{"stream": "deals", "data": {"id": "3980673856", "properties": {"amount": 60000, "amount_in_home_currency": 60000, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2014-08-31T00:00:00+00:00", "createdate": "2021-01-13T10:31:51.154000+00:00", "days_to_close": 0, "dealname": "Tim's Newer Deal", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": null, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_2": null, "hs_arr": null, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-13T10:31:51.154000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-01-13T10:31:51.154000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_days_to_close_raw": 0, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": null, "hs_exchange_rate": null, "hs_forecast_amount": 60000, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_is_open_count": 1, "hs_lastmodifieddate": "2021-09-07T18:11:59.757000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": null, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": 0, "hs_num_target_accounts": null, "hs_object_id": 3980673856, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 12000.0, "hs_projected_amount_in_home_currency": 12000.0, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": null, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 92113051787, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_v2_cumulative_time_in_66894120": null, "hs_v2_cumulative_time_in_9567448": null, "hs_v2_cumulative_time_in_9567449": null, "hs_v2_cumulative_time_in_customclosedwonstage": null, "hs_v2_date_entered_66894120": null, "hs_v2_date_entered_9567448": null, "hs_v2_date_entered_9567449": null, "hs_v2_date_entered_customclosedwonstage": null, "hs_v2_date_exited_66894120": null, "hs_v2_date_exited_9567448": null, "hs_v2_date_exited_9567449": null, "hs_v2_date_exited_customclosedwonstage": null, "hs_v2_latest_time_in_66894120": null, "hs_v2_latest_time_in_9567448": null, "hs_v2_latest_time_in_9567449": null, "hs_v2_latest_time_in_customclosedwonstage": null, "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-01-13T10:31:51.154000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-01-13T10:31:51.154Z", "updatedAt": "2021-09-07T18:11:59.757Z", "archived": false, "properties_amount": 60000, "properties_amount_in_home_currency": 60000, "properties_closed_lost_reason": null, "properties_closed_won_reason": null, "properties_closedate": "2014-08-31T00:00:00+00:00", "properties_createdate": "2021-01-13T10:31:51.154000+00:00", "properties_days_to_close": 0, "properties_dealname": "Tim's Newer Deal", "properties_dealstage": "appointmentscheduled", "properties_dealtype": "newbusiness", "properties_description": null, "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_hs_acv": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_collaborator_owner_ids": null, "properties_hs_all_deal_split_owner_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_latest_source": null, "properties_hs_analytics_latest_source_company": null, "properties_hs_analytics_latest_source_contact": null, "properties_hs_analytics_latest_source_data_1": null, "properties_hs_analytics_latest_source_data_1_company": null, "properties_hs_analytics_latest_source_data_1_contact": null, "properties_hs_analytics_latest_source_data_2": null, "properties_hs_analytics_latest_source_data_2_company": null, "properties_hs_analytics_latest_source_data_2_contact": null, "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_latest_source_timestamp_company": null, "properties_hs_analytics_latest_source_timestamp_contact": null, "properties_hs_analytics_source": null, "properties_hs_analytics_source_data_1": null, "properties_hs_analytics_source_data_2": null, "properties_hs_arr": null, "properties_hs_campaign": null, "properties_hs_closed_amount": 0, "properties_hs_closed_amount_in_home_currency": 0, "properties_hs_closed_won_count": null, "properties_hs_closed_won_date": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-13T10:31:51.154000+00:00", "properties_hs_date_entered_66894120": null, "properties_hs_date_entered_9567448": null, "properties_hs_date_entered_9567449": null, "properties_hs_date_entered_appointmentscheduled": "2021-01-13T10:31:51.154000+00:00", "properties_hs_date_entered_closedlost": null, "properties_hs_date_entered_closedwon": null, "properties_hs_date_entered_contractsent": null, "properties_hs_date_entered_customclosedwonstage": null, "properties_hs_date_entered_decisionmakerboughtin": null, "properties_hs_date_entered_presentationscheduled": null, "properties_hs_date_entered_qualifiedtobuy": null, "properties_hs_date_exited_66894120": null, "properties_hs_date_exited_9567448": null, "properties_hs_date_exited_9567449": null, "properties_hs_date_exited_appointmentscheduled": null, "properties_hs_date_exited_closedlost": null, "properties_hs_date_exited_closedwon": null, "properties_hs_date_exited_contractsent": null, "properties_hs_date_exited_customclosedwonstage": null, "properties_hs_date_exited_decisionmakerboughtin": null, "properties_hs_date_exited_presentationscheduled": null, "properties_hs_date_exited_qualifiedtobuy": null, "properties_hs_days_to_close_raw": 0, "properties_hs_deal_amount_calculation_preference": null, "properties_hs_deal_stage_probability": 0.2, "properties_hs_deal_stage_probability_shadow": null, "properties_hs_exchange_rate": null, "properties_hs_forecast_amount": 60000, "properties_hs_forecast_probability": null, "properties_hs_is_closed": false, "properties_hs_is_closed_won": false, "properties_hs_is_deal_split": false, "properties_hs_is_open_count": 1, "properties_hs_lastmodifieddate": "2021-09-07T18:11:59.757000+00:00", "properties_hs_latest_meeting_activity": null, "properties_hs_likelihood_to_close": null, "properties_hs_line_item_global_term_hs_discount_percentage": null, "properties_hs_line_item_global_term_hs_discount_percentage_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_period": null, "properties_hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "properties_hs_line_item_global_term_recurringbillingfrequency": null, "properties_hs_line_item_global_term_recurringbillingfrequency_enabled": null, "properties_hs_manual_forecast_category": null, "properties_hs_merged_object_ids": null, "properties_hs_mrr": null, "properties_hs_next_step": null, "properties_hs_num_associated_deal_splits": null, "properties_hs_num_of_associated_line_items": 0, "properties_hs_num_target_accounts": null, "properties_hs_object_id": 3980673856, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_predicted_amount": null, "properties_hs_predicted_amount_in_home_currency": null, "properties_hs_priority": null, "properties_hs_projected_amount": 12000.0, "properties_hs_projected_amount_in_home_currency": 12000.0, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_tcv": null, "properties_hs_time_in_66894120": null, "properties_hs_time_in_9567448": null, "properties_hs_time_in_9567449": null, "properties_hs_time_in_appointmentscheduled": 92113051787, "properties_hs_time_in_closedlost": null, "properties_hs_time_in_closedwon": null, "properties_hs_time_in_contractsent": null, "properties_hs_time_in_customclosedwonstage": null, "properties_hs_time_in_decisionmakerboughtin": null, "properties_hs_time_in_presentationscheduled": null, "properties_hs_time_in_qualifiedtobuy": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_v2_cumulative_time_in_66894120": null, "properties_hs_v2_cumulative_time_in_9567448": null, "properties_hs_v2_cumulative_time_in_9567449": null, "properties_hs_v2_cumulative_time_in_customclosedwonstage": null, "properties_hs_v2_date_entered_66894120": null, "properties_hs_v2_date_entered_9567448": null, "properties_hs_v2_date_entered_9567449": null, "properties_hs_v2_date_entered_customclosedwonstage": null, "properties_hs_v2_date_exited_66894120": null, "properties_hs_v2_date_exited_9567448": null, "properties_hs_v2_date_exited_9567449": null, "properties_hs_v2_date_exited_customclosedwonstage": null, "properties_hs_v2_latest_time_in_66894120": null, "properties_hs_v2_latest_time_in_9567448": null, "properties_hs_v2_latest_time_in_9567449": null, "properties_hs_v2_latest_time_in_customclosedwonstage": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2021-01-13T10:31:51.154000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_pipeline": "default"}, "emitted_at": 1702646964423} -{"stream": "deals", "data": {"id": "3986867076", "properties": {"amount": 6, "amount_in_home_currency": 6, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2014-08-31T00:00:00+00:00", "createdate": "2021-01-14T14:38:00.797000+00:00", "days_to_close": 0, "dealname": "Test Deal 2", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": null, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_2": null, "hs_arr": null, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-14T14:38:00.797000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-01-14T14:38:00.797000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_days_to_close_raw": 0, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": null, "hs_exchange_rate": null, "hs_forecast_amount": 6, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_is_open_count": 1, "hs_lastmodifieddate": "2021-09-07T00:24:18.932000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": null, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": 0, "hs_num_target_accounts": 0, "hs_object_id": 3986867076, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 1.2000000000000002, "hs_projected_amount_in_home_currency": 1.2000000000000002, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": null, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 92011882144, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_v2_cumulative_time_in_66894120": null, "hs_v2_cumulative_time_in_9567448": null, "hs_v2_cumulative_time_in_9567449": null, "hs_v2_cumulative_time_in_customclosedwonstage": null, "hs_v2_date_entered_66894120": null, "hs_v2_date_entered_9567448": null, "hs_v2_date_entered_9567449": null, "hs_v2_date_entered_customclosedwonstage": null, "hs_v2_date_exited_66894120": null, "hs_v2_date_exited_9567448": null, "hs_v2_date_exited_9567449": null, "hs_v2_date_exited_customclosedwonstage": null, "hs_v2_latest_time_in_66894120": null, "hs_v2_latest_time_in_9567448": null, "hs_v2_latest_time_in_9567449": null, "hs_v2_latest_time_in_customclosedwonstage": null, "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-01-14T14:38:00.797000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-01-14T14:38:00.797Z", "updatedAt": "2021-09-07T00:24:18.932Z", "archived": false, "companies": ["5183409178", "5183409178"], "properties_amount": 6, "properties_amount_in_home_currency": 6, "properties_closed_lost_reason": null, "properties_closed_won_reason": null, "properties_closedate": "2014-08-31T00:00:00+00:00", "properties_createdate": "2021-01-14T14:38:00.797000+00:00", "properties_days_to_close": 0, "properties_dealname": "Test Deal 2", "properties_dealstage": "appointmentscheduled", "properties_dealtype": "newbusiness", "properties_description": null, "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_hs_acv": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_collaborator_owner_ids": null, "properties_hs_all_deal_split_owner_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_latest_source": null, "properties_hs_analytics_latest_source_company": null, "properties_hs_analytics_latest_source_contact": null, "properties_hs_analytics_latest_source_data_1": null, "properties_hs_analytics_latest_source_data_1_company": null, "properties_hs_analytics_latest_source_data_1_contact": null, "properties_hs_analytics_latest_source_data_2": null, "properties_hs_analytics_latest_source_data_2_company": null, "properties_hs_analytics_latest_source_data_2_contact": null, "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_latest_source_timestamp_company": null, "properties_hs_analytics_latest_source_timestamp_contact": null, "properties_hs_analytics_source": null, "properties_hs_analytics_source_data_1": null, "properties_hs_analytics_source_data_2": null, "properties_hs_arr": null, "properties_hs_campaign": null, "properties_hs_closed_amount": 0, "properties_hs_closed_amount_in_home_currency": 0, "properties_hs_closed_won_count": null, "properties_hs_closed_won_date": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-14T14:38:00.797000+00:00", "properties_hs_date_entered_66894120": null, "properties_hs_date_entered_9567448": null, "properties_hs_date_entered_9567449": null, "properties_hs_date_entered_appointmentscheduled": "2021-01-14T14:38:00.797000+00:00", "properties_hs_date_entered_closedlost": null, "properties_hs_date_entered_closedwon": null, "properties_hs_date_entered_contractsent": null, "properties_hs_date_entered_customclosedwonstage": null, "properties_hs_date_entered_decisionmakerboughtin": null, "properties_hs_date_entered_presentationscheduled": null, "properties_hs_date_entered_qualifiedtobuy": null, "properties_hs_date_exited_66894120": null, "properties_hs_date_exited_9567448": null, "properties_hs_date_exited_9567449": null, "properties_hs_date_exited_appointmentscheduled": null, "properties_hs_date_exited_closedlost": null, "properties_hs_date_exited_closedwon": null, "properties_hs_date_exited_contractsent": null, "properties_hs_date_exited_customclosedwonstage": null, "properties_hs_date_exited_decisionmakerboughtin": null, "properties_hs_date_exited_presentationscheduled": null, "properties_hs_date_exited_qualifiedtobuy": null, "properties_hs_days_to_close_raw": 0, "properties_hs_deal_amount_calculation_preference": null, "properties_hs_deal_stage_probability": 0.2, "properties_hs_deal_stage_probability_shadow": null, "properties_hs_exchange_rate": null, "properties_hs_forecast_amount": 6, "properties_hs_forecast_probability": null, "properties_hs_is_closed": false, "properties_hs_is_closed_won": false, "properties_hs_is_deal_split": false, "properties_hs_is_open_count": 1, "properties_hs_lastmodifieddate": "2021-09-07T00:24:18.932000+00:00", "properties_hs_latest_meeting_activity": null, "properties_hs_likelihood_to_close": null, "properties_hs_line_item_global_term_hs_discount_percentage": null, "properties_hs_line_item_global_term_hs_discount_percentage_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_period": null, "properties_hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "properties_hs_line_item_global_term_recurringbillingfrequency": null, "properties_hs_line_item_global_term_recurringbillingfrequency_enabled": null, "properties_hs_manual_forecast_category": null, "properties_hs_merged_object_ids": null, "properties_hs_mrr": null, "properties_hs_next_step": null, "properties_hs_num_associated_deal_splits": null, "properties_hs_num_of_associated_line_items": 0, "properties_hs_num_target_accounts": 0, "properties_hs_object_id": 3986867076, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_predicted_amount": null, "properties_hs_predicted_amount_in_home_currency": null, "properties_hs_priority": null, "properties_hs_projected_amount": 1.2000000000000002, "properties_hs_projected_amount_in_home_currency": 1.2000000000000002, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_tcv": null, "properties_hs_time_in_66894120": null, "properties_hs_time_in_9567448": null, "properties_hs_time_in_9567449": null, "properties_hs_time_in_appointmentscheduled": 92011882144, "properties_hs_time_in_closedlost": null, "properties_hs_time_in_closedwon": null, "properties_hs_time_in_contractsent": null, "properties_hs_time_in_customclosedwonstage": null, "properties_hs_time_in_decisionmakerboughtin": null, "properties_hs_time_in_presentationscheduled": null, "properties_hs_time_in_qualifiedtobuy": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_v2_cumulative_time_in_66894120": null, "properties_hs_v2_cumulative_time_in_9567448": null, "properties_hs_v2_cumulative_time_in_9567449": null, "properties_hs_v2_cumulative_time_in_customclosedwonstage": null, "properties_hs_v2_date_entered_66894120": null, "properties_hs_v2_date_entered_9567448": null, "properties_hs_v2_date_entered_9567449": null, "properties_hs_v2_date_entered_customclosedwonstage": null, "properties_hs_v2_date_exited_66894120": null, "properties_hs_v2_date_exited_9567448": null, "properties_hs_v2_date_exited_9567449": null, "properties_hs_v2_date_exited_customclosedwonstage": null, "properties_hs_v2_latest_time_in_66894120": null, "properties_hs_v2_latest_time_in_9567448": null, "properties_hs_v2_latest_time_in_9567449": null, "properties_hs_v2_latest_time_in_customclosedwonstage": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2021-01-14T14:38:00.797000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_pipeline": "default"}, "emitted_at": 1702646964424} +{"stream": "deals", "data": {"id": "3980651569", "properties": {"amount": 60000, "amount_in_home_currency": 60000, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2014-08-31T00:00:00+00:00", "createdate": "2021-01-13T10:30:42.221000+00:00", "days_to_close": 0, "dealname": "Tim's Newer Deal", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": null, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": "OFFLINE", "hs_analytics_latest_source_company": "OFFLINE", "hs_analytics_latest_source_contact": null, "hs_analytics_latest_source_data_1": "CONTACTS", "hs_analytics_latest_source_data_1_company": "CONTACTS", "hs_analytics_latest_source_data_1_contact": null, "hs_analytics_latest_source_data_2": "CRM_UI", "hs_analytics_latest_source_data_2_company": "CRM_UI", "hs_analytics_latest_source_data_2_contact": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_2": "CRM_UI", "hs_arr": null, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-13T10:30:42.221000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-01-13T10:30:42.221000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_days_to_close_raw": 0, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": null, "hs_exchange_rate": null, "hs_forecast_amount": 60000, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_is_open_count": 1, "hs_lastmodifieddate": "2024-01-21T22:30:34.782000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": null, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": 0, "hs_num_target_accounts": 0, "hs_object_id": 3980651569, "hs_object_source": "API", "hs_object_source_id": null, "hs_object_source_label": "INTERNAL_PROCESSING", "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 12000.0, "hs_projected_amount_in_home_currency": 12000.0, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": null, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 95654671212, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-01-13T10:30:42.221000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-01-13T10:30:42.221Z", "updatedAt": "2024-01-21T22:30:34.782Z", "archived": false, "companies": ["5000526215", "5000526215"], "properties_amount": 60000, "properties_amount_in_home_currency": 60000, "properties_closed_lost_reason": null, "properties_closed_won_reason": null, "properties_closedate": "2014-08-31T00:00:00+00:00", "properties_createdate": "2021-01-13T10:30:42.221000+00:00", "properties_days_to_close": 0, "properties_dealname": "Tim's Newer Deal", "properties_dealstage": "appointmentscheduled", "properties_dealtype": "newbusiness", "properties_description": null, "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_hs_acv": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_collaborator_owner_ids": null, "properties_hs_all_deal_split_owner_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_latest_source": "OFFLINE", "properties_hs_analytics_latest_source_company": "OFFLINE", "properties_hs_analytics_latest_source_contact": null, "properties_hs_analytics_latest_source_data_1": "CONTACTS", "properties_hs_analytics_latest_source_data_1_company": "CONTACTS", "properties_hs_analytics_latest_source_data_1_contact": null, "properties_hs_analytics_latest_source_data_2": "CRM_UI", "properties_hs_analytics_latest_source_data_2_company": "CRM_UI", "properties_hs_analytics_latest_source_data_2_contact": null, "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_latest_source_timestamp_company": null, "properties_hs_analytics_latest_source_timestamp_contact": null, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "CONTACTS", "properties_hs_analytics_source_data_2": "CRM_UI", "properties_hs_arr": null, "properties_hs_campaign": null, "properties_hs_closed_amount": 0, "properties_hs_closed_amount_in_home_currency": 0, "properties_hs_closed_won_count": null, "properties_hs_closed_won_date": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-13T10:30:42.221000+00:00", "properties_hs_date_entered_66894120": null, "properties_hs_date_entered_9567448": null, "properties_hs_date_entered_9567449": null, "properties_hs_date_entered_appointmentscheduled": "2021-01-13T10:30:42.221000+00:00", "properties_hs_date_entered_closedlost": null, "properties_hs_date_entered_closedwon": null, "properties_hs_date_entered_contractsent": null, "properties_hs_date_entered_customclosedwonstage": null, "properties_hs_date_entered_decisionmakerboughtin": null, "properties_hs_date_entered_presentationscheduled": null, "properties_hs_date_entered_qualifiedtobuy": null, "properties_hs_date_exited_66894120": null, "properties_hs_date_exited_9567448": null, "properties_hs_date_exited_9567449": null, "properties_hs_date_exited_appointmentscheduled": null, "properties_hs_date_exited_closedlost": null, "properties_hs_date_exited_closedwon": null, "properties_hs_date_exited_contractsent": null, "properties_hs_date_exited_customclosedwonstage": null, "properties_hs_date_exited_decisionmakerboughtin": null, "properties_hs_date_exited_presentationscheduled": null, "properties_hs_date_exited_qualifiedtobuy": null, "properties_hs_days_to_close_raw": 0, "properties_hs_deal_amount_calculation_preference": null, "properties_hs_deal_stage_probability": 0.2, "properties_hs_deal_stage_probability_shadow": null, "properties_hs_exchange_rate": null, "properties_hs_forecast_amount": 60000, "properties_hs_forecast_probability": null, "properties_hs_is_closed": false, "properties_hs_is_closed_won": false, "properties_hs_is_deal_split": false, "properties_hs_is_open_count": 1, "properties_hs_lastmodifieddate": "2024-01-21T22:30:34.782000+00:00", "properties_hs_latest_meeting_activity": null, "properties_hs_likelihood_to_close": null, "properties_hs_line_item_global_term_hs_discount_percentage": null, "properties_hs_line_item_global_term_hs_discount_percentage_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_period": null, "properties_hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "properties_hs_line_item_global_term_recurringbillingfrequency": null, "properties_hs_line_item_global_term_recurringbillingfrequency_enabled": null, "properties_hs_manual_forecast_category": null, "properties_hs_merged_object_ids": null, "properties_hs_mrr": null, "properties_hs_next_step": null, "properties_hs_num_associated_deal_splits": null, "properties_hs_num_of_associated_line_items": 0, "properties_hs_num_target_accounts": 0, "properties_hs_object_id": 3980651569, "properties_hs_object_source": "API", "properties_hs_object_source_id": null, "properties_hs_object_source_label": "INTERNAL_PROCESSING", "properties_hs_object_source_user_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_predicted_amount": null, "properties_hs_predicted_amount_in_home_currency": null, "properties_hs_priority": null, "properties_hs_projected_amount": 12000.0, "properties_hs_projected_amount_in_home_currency": 12000.0, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_tcv": null, "properties_hs_time_in_66894120": null, "properties_hs_time_in_9567448": null, "properties_hs_time_in_9567449": null, "properties_hs_time_in_appointmentscheduled": 95654671212, "properties_hs_time_in_closedlost": null, "properties_hs_time_in_closedwon": null, "properties_hs_time_in_contractsent": null, "properties_hs_time_in_customclosedwonstage": null, "properties_hs_time_in_decisionmakerboughtin": null, "properties_hs_time_in_presentationscheduled": null, "properties_hs_time_in_qualifiedtobuy": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2021-01-13T10:30:42.221000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_pipeline": "default"}, "emitted_at": 1706188513541} +{"stream": "deals", "data": {"id": "3980673856", "properties": {"amount": 60000, "amount_in_home_currency": 60000, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2014-08-31T00:00:00+00:00", "createdate": "2021-01-13T10:31:51.154000+00:00", "days_to_close": 0, "dealname": "Tim's Newer Deal", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": null, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_2": null, "hs_arr": null, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-13T10:31:51.154000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-01-13T10:31:51.154000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_days_to_close_raw": 0, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": null, "hs_exchange_rate": null, "hs_forecast_amount": 60000, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_is_open_count": 1, "hs_lastmodifieddate": "2024-01-21T02:48:34.022000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": null, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": 0, "hs_num_target_accounts": null, "hs_object_id": 3980673856, "hs_object_source": "API", "hs_object_source_id": null, "hs_object_source_label": "INTERNAL_PROCESSING", "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 12000.0, "hs_projected_amount_in_home_currency": 12000.0, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": null, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 95654602279, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-01-13T10:31:51.154000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-01-13T10:31:51.154Z", "updatedAt": "2024-01-21T02:48:34.022Z", "archived": false, "properties_amount": 60000, "properties_amount_in_home_currency": 60000, "properties_closed_lost_reason": null, "properties_closed_won_reason": null, "properties_closedate": "2014-08-31T00:00:00+00:00", "properties_createdate": "2021-01-13T10:31:51.154000+00:00", "properties_days_to_close": 0, "properties_dealname": "Tim's Newer Deal", "properties_dealstage": "appointmentscheduled", "properties_dealtype": "newbusiness", "properties_description": null, "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_hs_acv": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_collaborator_owner_ids": null, "properties_hs_all_deal_split_owner_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_latest_source": null, "properties_hs_analytics_latest_source_company": null, "properties_hs_analytics_latest_source_contact": null, "properties_hs_analytics_latest_source_data_1": null, "properties_hs_analytics_latest_source_data_1_company": null, "properties_hs_analytics_latest_source_data_1_contact": null, "properties_hs_analytics_latest_source_data_2": null, "properties_hs_analytics_latest_source_data_2_company": null, "properties_hs_analytics_latest_source_data_2_contact": null, "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_latest_source_timestamp_company": null, "properties_hs_analytics_latest_source_timestamp_contact": null, "properties_hs_analytics_source": null, "properties_hs_analytics_source_data_1": null, "properties_hs_analytics_source_data_2": null, "properties_hs_arr": null, "properties_hs_campaign": null, "properties_hs_closed_amount": 0, "properties_hs_closed_amount_in_home_currency": 0, "properties_hs_closed_won_count": null, "properties_hs_closed_won_date": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-13T10:31:51.154000+00:00", "properties_hs_date_entered_66894120": null, "properties_hs_date_entered_9567448": null, "properties_hs_date_entered_9567449": null, "properties_hs_date_entered_appointmentscheduled": "2021-01-13T10:31:51.154000+00:00", "properties_hs_date_entered_closedlost": null, "properties_hs_date_entered_closedwon": null, "properties_hs_date_entered_contractsent": null, "properties_hs_date_entered_customclosedwonstage": null, "properties_hs_date_entered_decisionmakerboughtin": null, "properties_hs_date_entered_presentationscheduled": null, "properties_hs_date_entered_qualifiedtobuy": null, "properties_hs_date_exited_66894120": null, "properties_hs_date_exited_9567448": null, "properties_hs_date_exited_9567449": null, "properties_hs_date_exited_appointmentscheduled": null, "properties_hs_date_exited_closedlost": null, "properties_hs_date_exited_closedwon": null, "properties_hs_date_exited_contractsent": null, "properties_hs_date_exited_customclosedwonstage": null, "properties_hs_date_exited_decisionmakerboughtin": null, "properties_hs_date_exited_presentationscheduled": null, "properties_hs_date_exited_qualifiedtobuy": null, "properties_hs_days_to_close_raw": 0, "properties_hs_deal_amount_calculation_preference": null, "properties_hs_deal_stage_probability": 0.2, "properties_hs_deal_stage_probability_shadow": null, "properties_hs_exchange_rate": null, "properties_hs_forecast_amount": 60000, "properties_hs_forecast_probability": null, "properties_hs_is_closed": false, "properties_hs_is_closed_won": false, "properties_hs_is_deal_split": false, "properties_hs_is_open_count": 1, "properties_hs_lastmodifieddate": "2024-01-21T02:48:34.022000+00:00", "properties_hs_latest_meeting_activity": null, "properties_hs_likelihood_to_close": null, "properties_hs_line_item_global_term_hs_discount_percentage": null, "properties_hs_line_item_global_term_hs_discount_percentage_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_period": null, "properties_hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "properties_hs_line_item_global_term_recurringbillingfrequency": null, "properties_hs_line_item_global_term_recurringbillingfrequency_enabled": null, "properties_hs_manual_forecast_category": null, "properties_hs_merged_object_ids": null, "properties_hs_mrr": null, "properties_hs_next_step": null, "properties_hs_num_associated_deal_splits": null, "properties_hs_num_of_associated_line_items": 0, "properties_hs_num_target_accounts": null, "properties_hs_object_id": 3980673856, "properties_hs_object_source": "API", "properties_hs_object_source_id": null, "properties_hs_object_source_label": "INTERNAL_PROCESSING", "properties_hs_object_source_user_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_predicted_amount": null, "properties_hs_predicted_amount_in_home_currency": null, "properties_hs_priority": null, "properties_hs_projected_amount": 12000.0, "properties_hs_projected_amount_in_home_currency": 12000.0, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_tcv": null, "properties_hs_time_in_66894120": null, "properties_hs_time_in_9567448": null, "properties_hs_time_in_9567449": null, "properties_hs_time_in_appointmentscheduled": 95654602279, "properties_hs_time_in_closedlost": null, "properties_hs_time_in_closedwon": null, "properties_hs_time_in_contractsent": null, "properties_hs_time_in_customclosedwonstage": null, "properties_hs_time_in_decisionmakerboughtin": null, "properties_hs_time_in_presentationscheduled": null, "properties_hs_time_in_qualifiedtobuy": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2021-01-13T10:31:51.154000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_pipeline": "default"}, "emitted_at": 1706188513543} +{"stream": "deals", "data": {"id": "3986867076", "properties": {"amount": 6, "amount_in_home_currency": 6, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2014-08-31T00:00:00+00:00", "createdate": "2021-01-14T14:38:00.797000+00:00", "days_to_close": 0, "dealname": "Test Deal 2", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": null, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_2": null, "hs_arr": null, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-14T14:38:00.797000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-01-14T14:38:00.797000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_days_to_close_raw": 0, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": null, "hs_exchange_rate": null, "hs_forecast_amount": 6, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_is_open_count": 1, "hs_lastmodifieddate": "2024-01-20T00:59:40.882000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": null, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": 0, "hs_num_target_accounts": 0, "hs_object_id": 3986867076, "hs_object_source": "API", "hs_object_source_id": null, "hs_object_source_label": "INTERNAL_PROCESSING", "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 1.2000000000000002, "hs_projected_amount_in_home_currency": 1.2000000000000002, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": null, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 95553432636, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-01-14T14:38:00.797000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-01-14T14:38:00.797Z", "updatedAt": "2024-01-20T00:59:40.882Z", "archived": false, "companies": ["5183409178", "5183409178"], "properties_amount": 6, "properties_amount_in_home_currency": 6, "properties_closed_lost_reason": null, "properties_closed_won_reason": null, "properties_closedate": "2014-08-31T00:00:00+00:00", "properties_createdate": "2021-01-14T14:38:00.797000+00:00", "properties_days_to_close": 0, "properties_dealname": "Test Deal 2", "properties_dealstage": "appointmentscheduled", "properties_dealtype": "newbusiness", "properties_description": null, "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_hs_acv": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_collaborator_owner_ids": null, "properties_hs_all_deal_split_owner_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_latest_source": null, "properties_hs_analytics_latest_source_company": null, "properties_hs_analytics_latest_source_contact": null, "properties_hs_analytics_latest_source_data_1": null, "properties_hs_analytics_latest_source_data_1_company": null, "properties_hs_analytics_latest_source_data_1_contact": null, "properties_hs_analytics_latest_source_data_2": null, "properties_hs_analytics_latest_source_data_2_company": null, "properties_hs_analytics_latest_source_data_2_contact": null, "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_latest_source_timestamp_company": null, "properties_hs_analytics_latest_source_timestamp_contact": null, "properties_hs_analytics_source": null, "properties_hs_analytics_source_data_1": null, "properties_hs_analytics_source_data_2": null, "properties_hs_arr": null, "properties_hs_campaign": null, "properties_hs_closed_amount": 0, "properties_hs_closed_amount_in_home_currency": 0, "properties_hs_closed_won_count": null, "properties_hs_closed_won_date": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-14T14:38:00.797000+00:00", "properties_hs_date_entered_66894120": null, "properties_hs_date_entered_9567448": null, "properties_hs_date_entered_9567449": null, "properties_hs_date_entered_appointmentscheduled": "2021-01-14T14:38:00.797000+00:00", "properties_hs_date_entered_closedlost": null, "properties_hs_date_entered_closedwon": null, "properties_hs_date_entered_contractsent": null, "properties_hs_date_entered_customclosedwonstage": null, "properties_hs_date_entered_decisionmakerboughtin": null, "properties_hs_date_entered_presentationscheduled": null, "properties_hs_date_entered_qualifiedtobuy": null, "properties_hs_date_exited_66894120": null, "properties_hs_date_exited_9567448": null, "properties_hs_date_exited_9567449": null, "properties_hs_date_exited_appointmentscheduled": null, "properties_hs_date_exited_closedlost": null, "properties_hs_date_exited_closedwon": null, "properties_hs_date_exited_contractsent": null, "properties_hs_date_exited_customclosedwonstage": null, "properties_hs_date_exited_decisionmakerboughtin": null, "properties_hs_date_exited_presentationscheduled": null, "properties_hs_date_exited_qualifiedtobuy": null, "properties_hs_days_to_close_raw": 0, "properties_hs_deal_amount_calculation_preference": null, "properties_hs_deal_stage_probability": 0.2, "properties_hs_deal_stage_probability_shadow": null, "properties_hs_exchange_rate": null, "properties_hs_forecast_amount": 6, "properties_hs_forecast_probability": null, "properties_hs_is_closed": false, "properties_hs_is_closed_won": false, "properties_hs_is_deal_split": false, "properties_hs_is_open_count": 1, "properties_hs_lastmodifieddate": "2024-01-20T00:59:40.882000+00:00", "properties_hs_latest_meeting_activity": null, "properties_hs_likelihood_to_close": null, "properties_hs_line_item_global_term_hs_discount_percentage": null, "properties_hs_line_item_global_term_hs_discount_percentage_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_period": null, "properties_hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "properties_hs_line_item_global_term_recurringbillingfrequency": null, "properties_hs_line_item_global_term_recurringbillingfrequency_enabled": null, "properties_hs_manual_forecast_category": null, "properties_hs_merged_object_ids": null, "properties_hs_mrr": null, "properties_hs_next_step": null, "properties_hs_num_associated_deal_splits": null, "properties_hs_num_of_associated_line_items": 0, "properties_hs_num_target_accounts": 0, "properties_hs_object_id": 3986867076, "properties_hs_object_source": "API", "properties_hs_object_source_id": null, "properties_hs_object_source_label": "INTERNAL_PROCESSING", "properties_hs_object_source_user_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_predicted_amount": null, "properties_hs_predicted_amount_in_home_currency": null, "properties_hs_priority": null, "properties_hs_projected_amount": 1.2000000000000002, "properties_hs_projected_amount_in_home_currency": 1.2000000000000002, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_tcv": null, "properties_hs_time_in_66894120": null, "properties_hs_time_in_9567448": null, "properties_hs_time_in_9567449": null, "properties_hs_time_in_appointmentscheduled": 95553432636, "properties_hs_time_in_closedlost": null, "properties_hs_time_in_closedwon": null, "properties_hs_time_in_contractsent": null, "properties_hs_time_in_customclosedwonstage": null, "properties_hs_time_in_decisionmakerboughtin": null, "properties_hs_time_in_presentationscheduled": null, "properties_hs_time_in_qualifiedtobuy": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2021-01-14T14:38:00.797000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_pipeline": "default"}, "emitted_at": 1706188513544} {"stream": "email_events", "data": {"appName": "BatchTest", "location": {"country": "Unknown", "state": "Unknown", "city": "Unknown", "zipcode": "Unknown"}, "id": "17d3fcc4-bc34-38b4-9103-69b5896bbdde", "duration": 0, "browser": {"name": "Google Image Cache", "family": "Google Image Cache", "producer": "", "producerUrl": "", "type": "Proxy", "url": "", "version": []}, "created": 1614191191202, "userAgent": "Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)", "deviceType": "COMPUTER", "type": "OPEN", "recipient": "integration-test@airbyte.io", "portalId": 8727216, "sentBy": {"id": "dd239309-7866-4705-a3e9-c571dd349477", "created": 1614119023182}, "smtpId": null, "filteredEvent": false, "appId": 20053, "emailCampaignId": 2}, "emitted_at": 1697714199237} {"stream": "email_events", "data": {"appName": "BatchTest", "location": {"country": "Unknown", "state": "Unknown", "city": "Unknown", "zipcode": "Unknown"}, "id": "e5cbe134-db76-32cb-9e82-9dafcbaf8b64", "duration": 0, "browser": {"name": "Google Image Cache", "family": "Google Image Cache", "producer": "", "producerUrl": "", "type": "Proxy", "url": "", "version": []}, "created": 1614122124339, "userAgent": "Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)", "deviceType": "COMPUTER", "type": "OPEN", "recipient": "integration-test@airbyte.io", "portalId": 8727216, "sentBy": {"id": "dd239309-7866-4705-a3e9-c571dd349477", "created": 1614119023182}, "smtpId": null, "filteredEvent": false, "appId": 20053, "emailCampaignId": 2}, "emitted_at": 1697714199238} {"stream": "email_events", "data": {"appName": "BatchTest", "location": {"country": "UNITED STATES", "state": "california", "city": "mountain view", "latitude": 37.40599, "longitude": -122.078514, "zipcode": "94043"}, "id": "35b79cd1-3527-3ae7-b316-be0bbf872839", "duration": 1229, "browser": {"name": "Microsoft Edge 12.246", "family": "Microsoft Edge", "producer": "Microsoft Corporation.", "producerUrl": "https://www.microsoft.com/about/", "type": "Browser", "url": "https://en.wikipedia.org/wiki/Microsoft_Edge", "version": ["12.246"]}, "created": 1614119026757, "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0", "deviceType": "COMPUTER", "type": "OPEN", "recipient": "integration-test@airbyte.io", "portalId": 8727216, "sentBy": {"id": "dd239309-7866-4705-a3e9-c571dd349477", "created": 1614119023182}, "smtpId": null, "filteredEvent": true, "appId": 20053, "emailCampaignId": 2}, "emitted_at": 1697714199239} @@ -26,30 +26,30 @@ {"stream": "engagements", "data": {"id": 10584327028, "portalId": 8727216, "active": true, "createdAt": 1610636372009, "lastUpdated": 1610636372009, "type": "NOTE", "timestamp": 1409172644778, "allAccessibleTeamIds": [], "bodyPreview": "note body 5", "queueMembershipIds": [], "bodyPreviewIsTruncated": false, "bodyPreviewHtml": "\n \n \n note body 5\n \n", "gdprDeleted": false, "associations": {"contactIds": [], "companyIds": [], "dealIds": [], "ownerIds": [], "workflowIds": [], "ticketIds": [], "contentIds": [], "quoteIds": [], "marketingEventIds": []}, "attachments": [{"id": 4241968539}], "metadata": {"body": "note body 5"}, "associations_contactIds": [], "associations_companyIds": [], "associations_dealIds": [], "associations_ownerIds": [], "associations_workflowIds": [], "associations_ticketIds": [], "associations_contentIds": [], "associations_quoteIds": [], "associations_marketingEventIds": [], "metadata_body": "note body 5"}, "emitted_at": 1697714210187} {"stream": "engagements", "data": {"id": 10584327043, "portalId": 8727216, "active": true, "createdAt": 1610636372714, "lastUpdated": 1610636372714, "type": "NOTE", "timestamp": 1409172644778, "allAccessibleTeamIds": [], "bodyPreview": "note body 7", "queueMembershipIds": [], "bodyPreviewIsTruncated": false, "bodyPreviewHtml": "\n \n \n note body 7\n \n", "gdprDeleted": false, "associations": {"contactIds": [], "companyIds": [], "dealIds": [], "ownerIds": [], "workflowIds": [], "ticketIds": [], "contentIds": [], "quoteIds": [], "marketingEventIds": []}, "attachments": [{"id": 4241968539}], "metadata": {"body": "note body 7"}, "associations_contactIds": [], "associations_companyIds": [], "associations_dealIds": [], "associations_ownerIds": [], "associations_workflowIds": [], "associations_ticketIds": [], "associations_contentIds": [], "associations_quoteIds": [], "associations_marketingEventIds": [], "metadata_body": "note body 7"}, "emitted_at": 1697714210189} {"stream": "engagements", "data": {"id": 10584344127, "portalId": 8727216, "active": true, "createdAt": 1610636320990, "lastUpdated": 1610636320990, "type": "NOTE", "timestamp": 1409172644778, "allAccessibleTeamIds": [], "bodyPreview": "note body", "queueMembershipIds": [], "bodyPreviewIsTruncated": false, "bodyPreviewHtml": "\n \n \n note body\n \n", "gdprDeleted": false, "associations": {"contactIds": [], "companyIds": [], "dealIds": [], "ownerIds": [], "workflowIds": [], "ticketIds": [], "contentIds": [], "quoteIds": [], "marketingEventIds": []}, "attachments": [{"id": 4241968539}], "metadata": {"body": "note body"}, "associations_contactIds": [], "associations_companyIds": [], "associations_dealIds": [], "associations_ownerIds": [], "associations_workflowIds": [], "associations_ticketIds": [], "associations_contentIds": [], "associations_quoteIds": [], "associations_marketingEventIds": [], "metadata_body": "note body"}, "emitted_at": 1697714210190} -{"stream": "engagements_notes", "data": {"id": "10584327028", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": "4241968539", "hs_body_preview": "note body 5", "hs_body_preview_html": "\n \n \n note body 5\n \n", "hs_body_preview_is_truncated": false, "hs_created_by": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-14T14:59:32.009000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2021-01-14T14:59:32.009000+00:00", "hs_merged_object_ids": null, "hs_modified_by": null, "hs_note_body": "note body 5", "hs_object_id": 10584327028, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": "", "hubspot_team_id": null}, "createdAt": "2021-01-14T14:59:32.009Z", "updatedAt": "2021-01-14T14:59:32.009Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": "4241968539", "properties_hs_body_preview": "note body 5", "properties_hs_body_preview_html": "\n \n \n note body 5\n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_created_by": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-14T14:59:32.009000+00:00", "properties_hs_engagement_source": null, "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": false, "properties_hs_lastmodifieddate": "2021-01-14T14:59:32.009000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": null, "properties_hs_note_body": "note body 5", "properties_hs_object_id": 10584327028, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": "", "properties_hubspot_team_id": null}, "emitted_at": 1697714218669} -{"stream": "engagements_notes", "data": {"id": "10584327043", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": "4241968539", "hs_body_preview": "note body 7", "hs_body_preview_html": "\n \n \n note body 7\n \n", "hs_body_preview_is_truncated": false, "hs_created_by": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-14T14:59:32.714000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2021-01-14T14:59:32.714000+00:00", "hs_merged_object_ids": null, "hs_modified_by": null, "hs_note_body": "note body 7", "hs_object_id": 10584327043, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": "", "hubspot_team_id": null}, "createdAt": "2021-01-14T14:59:32.714Z", "updatedAt": "2021-01-14T14:59:32.714Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": "4241968539", "properties_hs_body_preview": "note body 7", "properties_hs_body_preview_html": "\n \n \n note body 7\n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_created_by": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-14T14:59:32.714000+00:00", "properties_hs_engagement_source": null, "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": false, "properties_hs_lastmodifieddate": "2021-01-14T14:59:32.714000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": null, "properties_hs_note_body": "note body 7", "properties_hs_object_id": 10584327043, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": "", "properties_hubspot_team_id": null}, "emitted_at": 1697714218670} -{"stream": "engagements_notes", "data": {"id": "10584344127", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": "4241968539", "hs_body_preview": "note body", "hs_body_preview_html": "\n \n \n note body\n \n", "hs_body_preview_is_truncated": false, "hs_created_by": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-14T14:58:40.990000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2021-01-14T14:58:40.990000+00:00", "hs_merged_object_ids": null, "hs_modified_by": null, "hs_note_body": "note body", "hs_object_id": 10584344127, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": "", "hubspot_team_id": null}, "createdAt": "2021-01-14T14:58:40.990Z", "updatedAt": "2021-01-14T14:58:40.990Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": "4241968539", "properties_hs_body_preview": "note body", "properties_hs_body_preview_html": "\n \n \n note body\n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_created_by": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-14T14:58:40.990000+00:00", "properties_hs_engagement_source": null, "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": false, "properties_hs_lastmodifieddate": "2021-01-14T14:58:40.990000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": null, "properties_hs_note_body": "note body", "properties_hs_object_id": 10584344127, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": "", "properties_hubspot_team_id": null}, "emitted_at": 1697714218671} -{"stream": "engagements_tasks", "data": {"id": "11257289597", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "hs_body_preview_html": "\n \n \n Regarding note logged on Tuesday, February 23, 2021 10:25 PM\n \n", "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2021-02-23T20:25:07.503000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2023-04-19T14:52:43.485000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_num_associated_companies": 0, "hs_num_associated_contacts": 0, "hs_num_associated_deals": 1, "hs_num_associated_queue_objects": 1, "hs_num_associated_tickets": 0, "hs_object_id": 11257289597, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[{\"engagementId\":11257289597,\"portalId\":8727216,\"engagementType\":\"TASK\",\"taskType\":\"REMINDER\",\"timestamp\":1614319200000,\"uuid\":\"TASK:e41fd851-f7c7-4381-85fa-796d076163aa\"}]}", "hs_task_body": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_is_overdue": true, "hs_task_is_past_due_date": true, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_missed_due_date": true, "hs_task_missed_due_date_count": 1, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": null, "hs_task_reminders": "1614319200000", "hs_task_repeat_interval": null, "hs_task_send_default_reminder": null, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "Follow up on Test deal 2", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_timestamp": "2021-02-26T06:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-02-23T20:25:07.503000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2021-02-23T20:25:07.503Z", "updatedAt": "2023-04-19T14:52:43.485Z", "archived": false, "deals": ["4315375411"], "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": null, "properties_hs_body_preview": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "properties_hs_body_preview_html": "\n \n \n Regarding note logged on Tuesday, February 23, 2021 10:25 PM\n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_calendar_event_id": null, "properties_hs_created_by": 12282590, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2021-02-23T20:25:07.503000+00:00", "properties_hs_engagement_source": null, "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": false, "properties_hs_lastmodifieddate": "2023-04-19T14:52:43.485000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": 12282590, "properties_hs_msteams_message_id": null, "properties_hs_num_associated_companies": 0, "properties_hs_num_associated_contacts": 0, "properties_hs_num_associated_deals": 1, "properties_hs_num_associated_queue_objects": 1, "properties_hs_num_associated_tickets": 0, "properties_hs_object_id": 11257289597, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_repeat_status": null, "properties_hs_scheduled_tasks": "{\"scheduledTasks\":[{\"engagementId\":11257289597,\"portalId\":8727216,\"engagementType\":\"TASK\",\"taskType\":\"REMINDER\",\"timestamp\":1614319200000,\"uuid\":\"TASK:e41fd851-f7c7-4381-85fa-796d076163aa\"}]}", "properties_hs_task_body": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "properties_hs_task_completion_count": null, "properties_hs_task_completion_date": null, "properties_hs_task_contact_timezone": null, "properties_hs_task_family": "SALES", "properties_hs_task_for_object_type": "OWNER", "properties_hs_task_is_all_day": false, "properties_hs_task_is_completed": 0, "properties_hs_task_is_completed_call": 0, "properties_hs_task_is_completed_email": 0, "properties_hs_task_is_completed_linked_in": 0, "properties_hs_task_is_completed_sequence": 0, "properties_hs_task_is_overdue": true, "properties_hs_task_is_past_due_date": true, "properties_hs_task_last_contact_outreach": null, "properties_hs_task_last_sales_activity_timestamp": null, "properties_hs_task_missed_due_date": true, "properties_hs_task_missed_due_date_count": 1, "properties_hs_task_priority": "NONE", "properties_hs_task_probability_to_complete": null, "properties_hs_task_relative_reminders": null, "properties_hs_task_reminders": "1614319200000", "properties_hs_task_repeat_interval": null, "properties_hs_task_send_default_reminder": null, "properties_hs_task_sequence_enrollment_active": null, "properties_hs_task_sequence_step_enrollment_id": null, "properties_hs_task_sequence_step_order": null, "properties_hs_task_status": "NOT_STARTED", "properties_hs_task_subject": "Follow up on Test deal 2", "properties_hs_task_template_id": null, "properties_hs_task_type": "TODO", "properties_hs_timestamp": "2021-02-26T06:00:00+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2021-02-23T20:25:07.503000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null}, "emitted_at": 1700237230220} -{"stream": "engagements_tasks", "data": {"id": "30652597343", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": null, "hs_body_preview_html": null, "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:41:48.834000+00:00", "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-04-04T15:11:47.231000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_num_associated_companies": 0, "hs_num_associated_contacts": 0, "hs_num_associated_deals": 0, "hs_num_associated_queue_objects": 0, "hs_num_associated_tickets": 0, "hs_object_id": 30652597343, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "hs_task_body": null, "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_is_overdue": true, "hs_task_is_past_due_date": true, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_missed_due_date": true, "hs_task_missed_due_date_count": 1, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": "[]", "hs_task_reminders": null, "hs_task_repeat_interval": null, "hs_task_send_default_reminder": false, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "test", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_timestamp": "2023-02-03T07:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:41:48.834000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:41:48.834Z", "updatedAt": "2023-04-04T15:11:47.231Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": null, "properties_hs_body_preview": null, "properties_hs_body_preview_html": null, "properties_hs_body_preview_is_truncated": false, "properties_hs_calendar_event_id": null, "properties_hs_created_by": 12282590, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-01-30T23:41:48.834000+00:00", "properties_hs_engagement_source": "CRM_UI", "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": null, "properties_hs_lastmodifieddate": "2023-04-04T15:11:47.231000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": 12282590, "properties_hs_msteams_message_id": null, "properties_hs_num_associated_companies": 0, "properties_hs_num_associated_contacts": 0, "properties_hs_num_associated_deals": 0, "properties_hs_num_associated_queue_objects": 0, "properties_hs_num_associated_tickets": 0, "properties_hs_object_id": 30652597343, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_repeat_status": null, "properties_hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "properties_hs_task_body": null, "properties_hs_task_completion_count": null, "properties_hs_task_completion_date": null, "properties_hs_task_contact_timezone": null, "properties_hs_task_family": "SALES", "properties_hs_task_for_object_type": "OWNER", "properties_hs_task_is_all_day": false, "properties_hs_task_is_completed": 0, "properties_hs_task_is_completed_call": 0, "properties_hs_task_is_completed_email": 0, "properties_hs_task_is_completed_linked_in": 0, "properties_hs_task_is_completed_sequence": 0, "properties_hs_task_is_overdue": true, "properties_hs_task_is_past_due_date": true, "properties_hs_task_last_contact_outreach": null, "properties_hs_task_last_sales_activity_timestamp": null, "properties_hs_task_missed_due_date": true, "properties_hs_task_missed_due_date_count": 1, "properties_hs_task_priority": "NONE", "properties_hs_task_probability_to_complete": null, "properties_hs_task_relative_reminders": "[]", "properties_hs_task_reminders": null, "properties_hs_task_repeat_interval": null, "properties_hs_task_send_default_reminder": false, "properties_hs_task_sequence_enrollment_active": null, "properties_hs_task_sequence_step_enrollment_id": null, "properties_hs_task_sequence_step_order": null, "properties_hs_task_status": "NOT_STARTED", "properties_hs_task_subject": "test", "properties_hs_task_template_id": null, "properties_hs_task_type": "TODO", "properties_hs_timestamp": "2023-02-03T07:00:00+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2023-01-30T23:41:48.834000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null}, "emitted_at": 1700237230222} -{"stream": "engagements_tasks", "data": {"id": "30652613208", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": null, "hs_body_preview_html": null, "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:51:52.099000+00:00", "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-01-30T23:51:54.343000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_num_associated_companies": 1, "hs_num_associated_contacts": 0, "hs_num_associated_deals": 0, "hs_num_associated_queue_objects": 1, "hs_num_associated_tickets": 0, "hs_object_id": 30652613208, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "hs_task_body": null, "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_is_overdue": true, "hs_task_is_past_due_date": true, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_missed_due_date": true, "hs_task_missed_due_date_count": 1, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": "[]", "hs_task_reminders": null, "hs_task_repeat_interval": null, "hs_task_send_default_reminder": false, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "test", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_timestamp": "2023-02-03T07:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:51:52.099000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:51:52.099Z", "updatedAt": "2023-01-30T23:51:54.343Z", "archived": false, "companies": ["11481383026"], "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": null, "properties_hs_body_preview": null, "properties_hs_body_preview_html": null, "properties_hs_body_preview_is_truncated": false, "properties_hs_calendar_event_id": null, "properties_hs_created_by": 12282590, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-01-30T23:51:52.099000+00:00", "properties_hs_engagement_source": "CRM_UI", "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": null, "properties_hs_lastmodifieddate": "2023-01-30T23:51:54.343000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": 12282590, "properties_hs_msteams_message_id": null, "properties_hs_num_associated_companies": 1, "properties_hs_num_associated_contacts": 0, "properties_hs_num_associated_deals": 0, "properties_hs_num_associated_queue_objects": 1, "properties_hs_num_associated_tickets": 0, "properties_hs_object_id": 30652613208, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_repeat_status": null, "properties_hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "properties_hs_task_body": null, "properties_hs_task_completion_count": null, "properties_hs_task_completion_date": null, "properties_hs_task_contact_timezone": null, "properties_hs_task_family": "SALES", "properties_hs_task_for_object_type": "OWNER", "properties_hs_task_is_all_day": false, "properties_hs_task_is_completed": 0, "properties_hs_task_is_completed_call": 0, "properties_hs_task_is_completed_email": 0, "properties_hs_task_is_completed_linked_in": 0, "properties_hs_task_is_completed_sequence": 0, "properties_hs_task_is_overdue": true, "properties_hs_task_is_past_due_date": true, "properties_hs_task_last_contact_outreach": null, "properties_hs_task_last_sales_activity_timestamp": null, "properties_hs_task_missed_due_date": true, "properties_hs_task_missed_due_date_count": 1, "properties_hs_task_priority": "NONE", "properties_hs_task_probability_to_complete": null, "properties_hs_task_relative_reminders": "[]", "properties_hs_task_reminders": null, "properties_hs_task_repeat_interval": null, "properties_hs_task_send_default_reminder": false, "properties_hs_task_sequence_enrollment_active": null, "properties_hs_task_sequence_step_enrollment_id": null, "properties_hs_task_sequence_step_order": null, "properties_hs_task_status": "NOT_STARTED", "properties_hs_task_subject": "test", "properties_hs_task_template_id": null, "properties_hs_task_type": "TODO", "properties_hs_timestamp": "2023-02-03T07:00:00+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2023-01-30T23:51:52.099000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null}, "emitted_at": 1700237230223} +{"stream": "engagements_notes", "data": {"id": "10584327028", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": "4241968539", "hs_body_preview": "note body 5", "hs_body_preview_html": "\n \n \n note body 5\n \n", "hs_body_preview_is_truncated": false, "hs_created_by": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-14T14:59:32.009000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2021-01-14T14:59:32.009000+00:00", "hs_merged_object_ids": null, "hs_modified_by": null, "hs_note_body": "note body 5", "hs_object_id": 10584327028, "hs_object_source": "API", "hs_object_source_id": null, "hs_object_source_label": "INTERNAL_PROCESSING", "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": "", "hubspot_team_id": null}, "createdAt": "2021-01-14T14:59:32.009Z", "updatedAt": "2021-01-14T14:59:32.009Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": "4241968539", "properties_hs_body_preview": "note body 5", "properties_hs_body_preview_html": "\n \n \n note body 5\n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_created_by": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-14T14:59:32.009000+00:00", "properties_hs_engagement_source": null, "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": false, "properties_hs_lastmodifieddate": "2021-01-14T14:59:32.009000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": null, "properties_hs_note_body": "note body 5", "properties_hs_object_id": 10584327028, "properties_hs_object_source": "API", "properties_hs_object_source_id": null, "properties_hs_object_source_label": "INTERNAL_PROCESSING", "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": "", "properties_hubspot_team_id": null}, "emitted_at": 1706189898297} +{"stream": "engagements_notes", "data": {"id": "10584327043", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": "4241968539", "hs_body_preview": "note body 7", "hs_body_preview_html": "\n \n \n note body 7\n \n", "hs_body_preview_is_truncated": false, "hs_created_by": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-14T14:59:32.714000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2021-01-14T14:59:32.714000+00:00", "hs_merged_object_ids": null, "hs_modified_by": null, "hs_note_body": "note body 7", "hs_object_id": 10584327043, "hs_object_source": "API", "hs_object_source_id": null, "hs_object_source_label": "INTERNAL_PROCESSING", "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": "", "hubspot_team_id": null}, "createdAt": "2021-01-14T14:59:32.714Z", "updatedAt": "2021-01-14T14:59:32.714Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": "4241968539", "properties_hs_body_preview": "note body 7", "properties_hs_body_preview_html": "\n \n \n note body 7\n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_created_by": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-14T14:59:32.714000+00:00", "properties_hs_engagement_source": null, "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": false, "properties_hs_lastmodifieddate": "2021-01-14T14:59:32.714000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": null, "properties_hs_note_body": "note body 7", "properties_hs_object_id": 10584327043, "properties_hs_object_source": "API", "properties_hs_object_source_id": null, "properties_hs_object_source_label": "INTERNAL_PROCESSING", "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": "", "properties_hubspot_team_id": null}, "emitted_at": 1706189898299} +{"stream": "engagements_notes", "data": {"id": "10584344127", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": "4241968539", "hs_body_preview": "note body", "hs_body_preview_html": "\n \n \n note body\n \n", "hs_body_preview_is_truncated": false, "hs_created_by": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-14T14:58:40.990000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2021-01-14T14:58:40.990000+00:00", "hs_merged_object_ids": null, "hs_modified_by": null, "hs_note_body": "note body", "hs_object_id": 10584344127, "hs_object_source": "API", "hs_object_source_id": null, "hs_object_source_label": "INTERNAL_PROCESSING", "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": "", "hubspot_team_id": null}, "createdAt": "2021-01-14T14:58:40.990Z", "updatedAt": "2021-01-14T14:58:40.990Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": "4241968539", "properties_hs_body_preview": "note body", "properties_hs_body_preview_html": "\n \n \n note body\n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_created_by": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-14T14:58:40.990000+00:00", "properties_hs_engagement_source": null, "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": false, "properties_hs_lastmodifieddate": "2021-01-14T14:58:40.990000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": null, "properties_hs_note_body": "note body", "properties_hs_object_id": 10584344127, "properties_hs_object_source": "API", "properties_hs_object_source_id": null, "properties_hs_object_source_label": "INTERNAL_PROCESSING", "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": "", "properties_hubspot_team_id": null}, "emitted_at": 1706189898299} +{"stream": "engagements_tasks", "data": {"id": "11257289597", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "hs_body_preview_html": "\n \n \n Regarding note logged on Tuesday, February 23, 2021 10:25 PM\n \n", "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2021-02-23T20:25:07.503000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2023-04-19T14:52:43.485000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_object_id": 11257289597, "hs_object_source": "CRM_UI", "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[{\"engagementId\":11257289597,\"portalId\":8727216,\"engagementType\":\"TASK\",\"taskType\":\"REMINDER\",\"timestamp\":1614319200000,\"uuid\":\"TASK:e41fd851-f7c7-4381-85fa-796d076163aa\"}]}", "hs_task_body": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_is_overdue": true, "hs_task_is_past_due_date": true, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_missed_due_date": true, "hs_task_missed_due_date_count": 1, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": null, "hs_task_reminders": "1614319200000", "hs_task_repeat_interval": null, "hs_task_send_default_reminder": null, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "Follow up on Test deal 2", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_timestamp": "2021-02-26T06:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-02-23T20:25:07.503000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2021-02-23T20:25:07.503Z", "updatedAt": "2023-04-19T14:52:43.485Z", "archived": false, "deals": ["4315375411"], "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": null, "properties_hs_body_preview": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "properties_hs_body_preview_html": "\n \n \n Regarding note logged on Tuesday, February 23, 2021 10:25 PM\n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_calendar_event_id": null, "properties_hs_created_by": 12282590, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2021-02-23T20:25:07.503000+00:00", "properties_hs_engagement_source": null, "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": false, "properties_hs_lastmodifieddate": "2023-04-19T14:52:43.485000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": 12282590, "properties_hs_msteams_message_id": null, "properties_hs_object_id": 11257289597, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_repeat_status": null, "properties_hs_scheduled_tasks": "{\"scheduledTasks\":[{\"engagementId\":11257289597,\"portalId\":8727216,\"engagementType\":\"TASK\",\"taskType\":\"REMINDER\",\"timestamp\":1614319200000,\"uuid\":\"TASK:e41fd851-f7c7-4381-85fa-796d076163aa\"}]}", "properties_hs_task_body": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "properties_hs_task_completion_count": null, "properties_hs_task_completion_date": null, "properties_hs_task_contact_timezone": null, "properties_hs_task_family": "SALES", "properties_hs_task_for_object_type": "OWNER", "properties_hs_task_is_all_day": false, "properties_hs_task_is_completed": 0, "properties_hs_task_is_completed_call": 0, "properties_hs_task_is_completed_email": 0, "properties_hs_task_is_completed_linked_in": 0, "properties_hs_task_is_completed_sequence": 0, "properties_hs_task_is_overdue": true, "properties_hs_task_is_past_due_date": true, "properties_hs_task_last_contact_outreach": null, "properties_hs_task_last_sales_activity_timestamp": null, "properties_hs_task_missed_due_date": true, "properties_hs_task_missed_due_date_count": 1, "properties_hs_task_priority": "NONE", "properties_hs_task_probability_to_complete": null, "properties_hs_task_relative_reminders": null, "properties_hs_task_reminders": "1614319200000", "properties_hs_task_repeat_interval": null, "properties_hs_task_send_default_reminder": null, "properties_hs_task_sequence_enrollment_active": null, "properties_hs_task_sequence_step_enrollment_id": null, "properties_hs_task_sequence_step_order": null, "properties_hs_task_status": "NOT_STARTED", "properties_hs_task_subject": "Follow up on Test deal 2", "properties_hs_task_template_id": null, "properties_hs_task_type": "TODO", "properties_hs_timestamp": "2021-02-26T06:00:00+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2021-02-23T20:25:07.503000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null}, "emitted_at": 1706697197260} +{"stream": "engagements_tasks", "data": {"id": "30652597343", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": null, "hs_body_preview_html": null, "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:41:48.834000+00:00", "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-04-04T15:11:47.231000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_object_id": 30652597343, "hs_object_source": "CRM_UI", "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "hs_task_body": null, "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_is_overdue": true, "hs_task_is_past_due_date": true, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_missed_due_date": true, "hs_task_missed_due_date_count": 1, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": "[]", "hs_task_reminders": null, "hs_task_repeat_interval": null, "hs_task_send_default_reminder": false, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "test", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_timestamp": "2023-02-03T07:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:41:48.834000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:41:48.834Z", "updatedAt": "2023-04-04T15:11:47.231Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": null, "properties_hs_body_preview": null, "properties_hs_body_preview_html": null, "properties_hs_body_preview_is_truncated": false, "properties_hs_calendar_event_id": null, "properties_hs_created_by": 12282590, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-01-30T23:41:48.834000+00:00", "properties_hs_engagement_source": "CRM_UI", "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": null, "properties_hs_lastmodifieddate": "2023-04-04T15:11:47.231000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": 12282590, "properties_hs_msteams_message_id": null, "properties_hs_object_id": 30652597343, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_repeat_status": null, "properties_hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "properties_hs_task_body": null, "properties_hs_task_completion_count": null, "properties_hs_task_completion_date": null, "properties_hs_task_contact_timezone": null, "properties_hs_task_family": "SALES", "properties_hs_task_for_object_type": "OWNER", "properties_hs_task_is_all_day": false, "properties_hs_task_is_completed": 0, "properties_hs_task_is_completed_call": 0, "properties_hs_task_is_completed_email": 0, "properties_hs_task_is_completed_linked_in": 0, "properties_hs_task_is_completed_sequence": 0, "properties_hs_task_is_overdue": true, "properties_hs_task_is_past_due_date": true, "properties_hs_task_last_contact_outreach": null, "properties_hs_task_last_sales_activity_timestamp": null, "properties_hs_task_missed_due_date": true, "properties_hs_task_missed_due_date_count": 1, "properties_hs_task_priority": "NONE", "properties_hs_task_probability_to_complete": null, "properties_hs_task_relative_reminders": "[]", "properties_hs_task_reminders": null, "properties_hs_task_repeat_interval": null, "properties_hs_task_send_default_reminder": false, "properties_hs_task_sequence_enrollment_active": null, "properties_hs_task_sequence_step_enrollment_id": null, "properties_hs_task_sequence_step_order": null, "properties_hs_task_status": "NOT_STARTED", "properties_hs_task_subject": "test", "properties_hs_task_template_id": null, "properties_hs_task_type": "TODO", "properties_hs_timestamp": "2023-02-03T07:00:00+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2023-01-30T23:41:48.834000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null}, "emitted_at": 1706697197261} +{"stream": "engagements_tasks", "data": {"id": "30652613208", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": null, "hs_body_preview_html": null, "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:51:52.099000+00:00", "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-01-30T23:51:54.343000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_object_id": 30652613208, "hs_object_source": "CRM_UI", "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "hs_task_body": null, "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_is_overdue": true, "hs_task_is_past_due_date": true, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_missed_due_date": true, "hs_task_missed_due_date_count": 1, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": "[]", "hs_task_reminders": null, "hs_task_repeat_interval": null, "hs_task_send_default_reminder": false, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "test", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_timestamp": "2023-02-03T07:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:51:52.099000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:51:52.099Z", "updatedAt": "2023-01-30T23:51:54.343Z", "archived": false, "companies": ["11481383026"], "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": null, "properties_hs_body_preview": null, "properties_hs_body_preview_html": null, "properties_hs_body_preview_is_truncated": false, "properties_hs_calendar_event_id": null, "properties_hs_created_by": 12282590, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-01-30T23:51:52.099000+00:00", "properties_hs_engagement_source": "CRM_UI", "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": null, "properties_hs_lastmodifieddate": "2023-01-30T23:51:54.343000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": 12282590, "properties_hs_msteams_message_id": null, "properties_hs_object_id": 30652613208, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_repeat_status": null, "properties_hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "properties_hs_task_body": null, "properties_hs_task_completion_count": null, "properties_hs_task_completion_date": null, "properties_hs_task_contact_timezone": null, "properties_hs_task_family": "SALES", "properties_hs_task_for_object_type": "OWNER", "properties_hs_task_is_all_day": false, "properties_hs_task_is_completed": 0, "properties_hs_task_is_completed_call": 0, "properties_hs_task_is_completed_email": 0, "properties_hs_task_is_completed_linked_in": 0, "properties_hs_task_is_completed_sequence": 0, "properties_hs_task_is_overdue": true, "properties_hs_task_is_past_due_date": true, "properties_hs_task_last_contact_outreach": null, "properties_hs_task_last_sales_activity_timestamp": null, "properties_hs_task_missed_due_date": true, "properties_hs_task_missed_due_date_count": 1, "properties_hs_task_priority": "NONE", "properties_hs_task_probability_to_complete": null, "properties_hs_task_relative_reminders": "[]", "properties_hs_task_reminders": null, "properties_hs_task_repeat_interval": null, "properties_hs_task_send_default_reminder": false, "properties_hs_task_sequence_enrollment_active": null, "properties_hs_task_sequence_step_enrollment_id": null, "properties_hs_task_sequence_step_order": null, "properties_hs_task_status": "NOT_STARTED", "properties_hs_task_subject": "test", "properties_hs_task_template_id": null, "properties_hs_task_type": "TODO", "properties_hs_timestamp": "2023-02-03T07:00:00+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2023-01-30T23:51:52.099000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null}, "emitted_at": 1706697197263} {"stream": "forms", "data": {"id": "01ba116c-f3a8-4957-8884-ff0c4420af76", "name": "DemoForm", "createdAt": "2021-01-14T14:44:48.278Z", "updatedAt": "2021-01-14T14:44:48.278Z", "archived": false, "fieldGroups": [{"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "firstname", "label": "First Name", "required": false, "hidden": false, "fieldType": "single_line_text"}]}, {"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "lastname", "label": "Last Name", "required": false, "hidden": false, "fieldType": "single_line_text"}]}, {"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "adress_1", "label": "Adress 1", "required": false, "hidden": false, "fieldType": "single_line_text"}]}], "configuration": {"language": "en", "cloneable": true, "postSubmitAction": {"type": "thank_you", "value": ""}, "editable": true, "archivable": true, "recaptchaEnabled": false, "notifyContactOwner": false, "notifyRecipients": [], "createNewContactForNewEmail": false, "prePopulateKnownValues": true, "allowLinkToResetKnownValues": false, "lifecycleStages": []}, "displayOptions": {"renderRawHtml": false, "theme": "default_style", "submitButtonText": "Submit", "style": {"fontFamily": "arial, helvetica, sans-serif", "backgroundWidth": "100%", "labelTextColor": "#33475b", "labelTextSize": "11px", "helpTextColor": "#7C98B6", "helpTextSize": "11px", "legalConsentTextColor": "#33475b", "legalConsentTextSize": "14px", "submitColor": "#ff7a59", "submitAlignment": "left", "submitFontColor": "#ffffff", "submitSize": "12px"}, "cssClass": null}, "legalConsentOptions": {"type": "none"}, "formType": "hubspot"}, "emitted_at": 1697714221520} {"stream": "forms", "data": {"id": "03e69987-1dcb-4d55-9cb6-d3812ac00ee6", "name": "New form 93", "createdAt": "2023-02-13T16:56:33.108Z", "updatedAt": "2023-02-13T16:56:33.108Z", "archived": false, "fieldGroups": [{"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "email", "label": "Email", "required": true, "hidden": false, "fieldType": "email", "validation": {"blockedEmailDomains": [], "useDefaultBlockList": false}}]}], "configuration": {"language": "en", "cloneable": true, "postSubmitAction": {"type": "thank_you", "value": "Thanks for submitting the form."}, "editable": true, "archivable": true, "recaptchaEnabled": false, "notifyContactOwner": false, "notifyRecipients": ["12282590"], "createNewContactForNewEmail": false, "prePopulateKnownValues": true, "allowLinkToResetKnownValues": false, "lifecycleStages": []}, "displayOptions": {"renderRawHtml": false, "theme": "default_style", "submitButtonText": "Submit", "style": {"fontFamily": "arial, helvetica, sans-serif", "backgroundWidth": "100%", "labelTextColor": "#33475b", "labelTextSize": "14px", "helpTextColor": "#7C98B6", "helpTextSize": "11px", "legalConsentTextColor": "#33475b", "legalConsentTextSize": "14px", "submitColor": "#ff7a59", "submitAlignment": "left", "submitFontColor": "#ffffff", "submitSize": "12px"}, "cssClass": "hs-form stacked"}, "legalConsentOptions": {"type": "implicit_consent_to_process", "communicationConsentText": "integrationtest is committed to protecting and respecting your privacy, and we\u2019ll only use your personal information to administer your account and to provide the products and services you requested from us. From time to time, we would like to contact you about our products and services, as well as other content that may be of interest to you. If you consent to us contacting you for this purpose, please tick below to say how you would like us to contact you:", "communicationsCheckboxes": [{"required": false, "subscriptionTypeId": 23704464, "label": "I agree to receive other communications from [MAIN] integration test account."}], "privacyText": "You may unsubscribe from these communications at any time. For more information on how to unsubscribe, our privacy practices, and how we are committed to protecting and respecting your privacy, please review our Privacy Policy.", "consentToProcessText": "By clicking submit below, you consent to allow integrationtest to store and process the personal information submitted above to provide you the content requested."}, "formType": "hubspot"}, "emitted_at": 1697714221521} {"stream": "forms", "data": {"id": "0a7fd84f-471e-444a-a4e0-ca36d39f8af7", "name": "New form 27", "createdAt": "2023-02-13T16:45:22.640Z", "updatedAt": "2023-02-13T16:45:22.640Z", "archived": false, "fieldGroups": [{"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "email", "label": "Email", "required": true, "hidden": false, "fieldType": "email", "validation": {"blockedEmailDomains": [], "useDefaultBlockList": false}}]}], "configuration": {"language": "en", "cloneable": true, "postSubmitAction": {"type": "thank_you", "value": "Thanks for submitting the form."}, "editable": true, "archivable": true, "recaptchaEnabled": false, "notifyContactOwner": false, "notifyRecipients": ["12282590"], "createNewContactForNewEmail": false, "prePopulateKnownValues": true, "allowLinkToResetKnownValues": false, "lifecycleStages": []}, "displayOptions": {"renderRawHtml": false, "theme": "default_style", "submitButtonText": "Submit", "style": {"fontFamily": "arial, helvetica, sans-serif", "backgroundWidth": "100%", "labelTextColor": "#33475b", "labelTextSize": "14px", "helpTextColor": "#7C98B6", "helpTextSize": "11px", "legalConsentTextColor": "#33475b", "legalConsentTextSize": "14px", "submitColor": "#ff7a59", "submitAlignment": "left", "submitFontColor": "#ffffff", "submitSize": "12px"}, "cssClass": "hs-form stacked"}, "legalConsentOptions": {"type": "implicit_consent_to_process", "communicationConsentText": "integrationtest is committed to protecting and respecting your privacy, and we\u2019ll only use your personal information to administer your account and to provide the products and services you requested from us. From time to time, we would like to contact you about our products and services, as well as other content that may be of interest to you. If you consent to us contacting you for this purpose, please tick below to say how you would like us to contact you:", "communicationsCheckboxes": [{"required": false, "subscriptionTypeId": 23704464, "label": "I agree to receive other communications from [MAIN] integration test account."}], "privacyText": "You may unsubscribe from these communications at any time. For more information on how to unsubscribe, our privacy practices, and how we are committed to protecting and respecting your privacy, please review our Privacy Policy.", "consentToProcessText": "By clicking submit below, you consent to allow integrationtest to store and process the personal information submitted above to provide you the content requested."}, "formType": "hubspot"}, "emitted_at": 1697714221522} -{"stream": "goals", "data": {"id": "221880757009", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-07-31T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "hs_kpi_is_team_rollup": false, "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-08-01T00:45:14.830000+00:00", "hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757009, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_outcome": "completed", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-07-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "achieved", "hs_status_display_order": 4, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_template_id": 4, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-12-11T20:46:14.473Z", "archived": false, "properties_hs__migration_soft_delete": null, "properties_hs_ad_account_asset_ids": null, "properties_hs_ad_campaign_asset_ids": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignee_team_id": null, "properties_hs_assignee_user_id": 26748728, "properties_hs_contact_lifecycle_stage": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-10T13:57:36.691000+00:00", "properties_hs_currency": null, "properties_hs_deal_pipeline_ids": null, "properties_hs_edit_updates_notification_frequency": "weekly", "properties_hs_end_date": null, "properties_hs_end_datetime": "2023-07-31T23:59:59.999000+00:00", "properties_hs_fiscal_year_offset": 0, "properties_hs_goal_name": "Integration Test Goal Hubspot", "properties_hs_goal_target_group_id": 221880750627, "properties_hs_goal_type": "average_ticket_response_time", "properties_hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "properties_hs_is_forecastable": "true", "properties_hs_is_legacy": null, "properties_hs_kpi_display_unit": "hour", "properties_hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "properties_hs_kpi_is_team_rollup": false, "properties_hs_kpi_metric_type": "AVG", "properties_hs_kpi_object_type": "TICKET", "properties_hs_kpi_object_type_id": "0-5", "properties_hs_kpi_progress_percent": null, "properties_hs_kpi_property_name": "time_to_first_agent_reply", "properties_hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "properties_hs_kpi_time_period_property": "createdate", "properties_hs_kpi_tracking_method": "LOWER_IS_BETTER", "properties_hs_kpi_unit_type": "duration", "properties_hs_kpi_value": 0.0, "properties_hs_kpi_value_calculated_at": null, "properties_hs_kpi_value_last_calculated_at": "2023-08-01T00:45:14.830000+00:00", "properties_hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "properties_hs_legacy_active": null, "properties_hs_legacy_created_at": null, "properties_hs_legacy_created_by": null, "properties_hs_legacy_quarterly_target_composite_id": null, "properties_hs_legacy_sql_id": null, "properties_hs_legacy_unique_sql_id": null, "properties_hs_legacy_updated_at": null, "properties_hs_legacy_updated_by": null, "properties_hs_merged_object_ids": null, "properties_hs_migration_soft_delete": null, "properties_hs_milestone": "monthly", "properties_hs_object_id": 221880757009, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_outcome": "completed", "properties_hs_owner_ids_of_all_owners": "111730024", "properties_hs_participant_type": "users", "properties_hs_pipelines": "0", "properties_hs_progress_updates_notification_frequency": "weekly", "properties_hs_read_only": null, "properties_hs_should_notify_on_achieved": "false", "properties_hs_should_notify_on_edit_updates": "false", "properties_hs_should_notify_on_exceeded": "false", "properties_hs_should_notify_on_kickoff": "false", "properties_hs_should_notify_on_missed": "false", "properties_hs_should_notify_on_progress_updates": "false", "properties_hs_should_recalculate": "false", "properties_hs_start_date": null, "properties_hs_start_datetime": "2023-07-01T00:00:00+00:00", "properties_hs_static_kpi_filter_groups": "[]", "properties_hs_status": "achieved", "properties_hs_status_display_order": 4, "properties_hs_target_amount": 0.0, "properties_hs_target_amount_in_home_currency": 0.0, "properties_hs_team_id": null, "properties_hs_template_id": 4, "properties_hs_ticket_pipeline_ids": "0", "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "26748728", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null}, "emitted_at": 1702410363120} -{"stream": "goals", "data": {"id": "221880757010", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-09-30T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "hs_kpi_is_team_rollup": false, "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-10-01T22:31:08.621000+00:00", "hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757010, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_outcome": "completed", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-09-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "achieved", "hs_status_display_order": 4, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_template_id": 4, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-12-11T20:46:14.473Z", "archived": false, "properties_hs__migration_soft_delete": null, "properties_hs_ad_account_asset_ids": null, "properties_hs_ad_campaign_asset_ids": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignee_team_id": null, "properties_hs_assignee_user_id": 26748728, "properties_hs_contact_lifecycle_stage": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-10T13:57:36.691000+00:00", "properties_hs_currency": null, "properties_hs_deal_pipeline_ids": null, "properties_hs_edit_updates_notification_frequency": "weekly", "properties_hs_end_date": null, "properties_hs_end_datetime": "2023-09-30T23:59:59.999000+00:00", "properties_hs_fiscal_year_offset": 0, "properties_hs_goal_name": "Integration Test Goal Hubspot", "properties_hs_goal_target_group_id": 221880750627, "properties_hs_goal_type": "average_ticket_response_time", "properties_hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "properties_hs_is_forecastable": "true", "properties_hs_is_legacy": null, "properties_hs_kpi_display_unit": "hour", "properties_hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "properties_hs_kpi_is_team_rollup": false, "properties_hs_kpi_metric_type": "AVG", "properties_hs_kpi_object_type": "TICKET", "properties_hs_kpi_object_type_id": "0-5", "properties_hs_kpi_progress_percent": null, "properties_hs_kpi_property_name": "time_to_first_agent_reply", "properties_hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "properties_hs_kpi_time_period_property": "createdate", "properties_hs_kpi_tracking_method": "LOWER_IS_BETTER", "properties_hs_kpi_unit_type": "duration", "properties_hs_kpi_value": 0.0, "properties_hs_kpi_value_calculated_at": null, "properties_hs_kpi_value_last_calculated_at": "2023-10-01T22:31:08.621000+00:00", "properties_hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "properties_hs_legacy_active": null, "properties_hs_legacy_created_at": null, "properties_hs_legacy_created_by": null, "properties_hs_legacy_quarterly_target_composite_id": null, "properties_hs_legacy_sql_id": null, "properties_hs_legacy_unique_sql_id": null, "properties_hs_legacy_updated_at": null, "properties_hs_legacy_updated_by": null, "properties_hs_merged_object_ids": null, "properties_hs_migration_soft_delete": null, "properties_hs_milestone": "monthly", "properties_hs_object_id": 221880757010, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_outcome": "completed", "properties_hs_owner_ids_of_all_owners": "111730024", "properties_hs_participant_type": "users", "properties_hs_pipelines": "0", "properties_hs_progress_updates_notification_frequency": "weekly", "properties_hs_read_only": null, "properties_hs_should_notify_on_achieved": "false", "properties_hs_should_notify_on_edit_updates": "false", "properties_hs_should_notify_on_exceeded": "false", "properties_hs_should_notify_on_kickoff": "false", "properties_hs_should_notify_on_missed": "false", "properties_hs_should_notify_on_progress_updates": "false", "properties_hs_should_recalculate": "false", "properties_hs_start_date": null, "properties_hs_start_datetime": "2023-09-01T00:00:00+00:00", "properties_hs_static_kpi_filter_groups": "[]", "properties_hs_status": "achieved", "properties_hs_status_display_order": 4, "properties_hs_target_amount": 0.0, "properties_hs_target_amount_in_home_currency": 0.0, "properties_hs_team_id": null, "properties_hs_template_id": 4, "properties_hs_ticket_pipeline_ids": "0", "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "26748728", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null}, "emitted_at": 1702410363124} -{"stream": "goals", "data": {"id": "221880757011", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-08-31T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "hs_kpi_is_team_rollup": false, "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-09-01T15:26:00.500000+00:00", "hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757011, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_outcome": "completed", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-08-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "achieved", "hs_status_display_order": 4, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_template_id": 4, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-12-11T20:46:14.473Z", "archived": false, "properties_hs__migration_soft_delete": null, "properties_hs_ad_account_asset_ids": null, "properties_hs_ad_campaign_asset_ids": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignee_team_id": null, "properties_hs_assignee_user_id": 26748728, "properties_hs_contact_lifecycle_stage": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-10T13:57:36.691000+00:00", "properties_hs_currency": null, "properties_hs_deal_pipeline_ids": null, "properties_hs_edit_updates_notification_frequency": "weekly", "properties_hs_end_date": null, "properties_hs_end_datetime": "2023-08-31T23:59:59.999000+00:00", "properties_hs_fiscal_year_offset": 0, "properties_hs_goal_name": "Integration Test Goal Hubspot", "properties_hs_goal_target_group_id": 221880750627, "properties_hs_goal_type": "average_ticket_response_time", "properties_hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "properties_hs_is_forecastable": "true", "properties_hs_is_legacy": null, "properties_hs_kpi_display_unit": "hour", "properties_hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "properties_hs_kpi_is_team_rollup": false, "properties_hs_kpi_metric_type": "AVG", "properties_hs_kpi_object_type": "TICKET", "properties_hs_kpi_object_type_id": "0-5", "properties_hs_kpi_progress_percent": null, "properties_hs_kpi_property_name": "time_to_first_agent_reply", "properties_hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "properties_hs_kpi_time_period_property": "createdate", "properties_hs_kpi_tracking_method": "LOWER_IS_BETTER", "properties_hs_kpi_unit_type": "duration", "properties_hs_kpi_value": 0.0, "properties_hs_kpi_value_calculated_at": null, "properties_hs_kpi_value_last_calculated_at": "2023-09-01T15:26:00.500000+00:00", "properties_hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "properties_hs_legacy_active": null, "properties_hs_legacy_created_at": null, "properties_hs_legacy_created_by": null, "properties_hs_legacy_quarterly_target_composite_id": null, "properties_hs_legacy_sql_id": null, "properties_hs_legacy_unique_sql_id": null, "properties_hs_legacy_updated_at": null, "properties_hs_legacy_updated_by": null, "properties_hs_merged_object_ids": null, "properties_hs_migration_soft_delete": null, "properties_hs_milestone": "monthly", "properties_hs_object_id": 221880757011, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_outcome": "completed", "properties_hs_owner_ids_of_all_owners": "111730024", "properties_hs_participant_type": "users", "properties_hs_pipelines": "0", "properties_hs_progress_updates_notification_frequency": "weekly", "properties_hs_read_only": null, "properties_hs_should_notify_on_achieved": "false", "properties_hs_should_notify_on_edit_updates": "false", "properties_hs_should_notify_on_exceeded": "false", "properties_hs_should_notify_on_kickoff": "false", "properties_hs_should_notify_on_missed": "false", "properties_hs_should_notify_on_progress_updates": "false", "properties_hs_should_recalculate": "false", "properties_hs_start_date": null, "properties_hs_start_datetime": "2023-08-01T00:00:00+00:00", "properties_hs_static_kpi_filter_groups": "[]", "properties_hs_status": "achieved", "properties_hs_status_display_order": 4, "properties_hs_target_amount": 0.0, "properties_hs_target_amount_in_home_currency": 0.0, "properties_hs_team_id": null, "properties_hs_template_id": 4, "properties_hs_ticket_pipeline_ids": "0", "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "26748728", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null}, "emitted_at": 1702410363125} -{"stream": "line_items", "data": {"id": "1188257165", "properties": {"amount": 10.0, "createdate": "2021-02-23T20:11:54.030000+00:00", "description": "Baseball hat, medium", "discount": null, "hs_acv": 10.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 0.0, "hs_billing_period_end_date": null, "hs_billing_period_start_date": null, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": 5, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2021-07-17T23:50:32.502000+00:00", "hs_line_item_currency_code": null, "hs_margin": 5.0, "hs_margin_acv": 5.0, "hs_margin_arr": 0.0, "hs_margin_mrr": 0.0, "hs_margin_tcv": 5.0, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_object_id": 1188257165, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_position_on_quote": 0, "hs_pre_discount_amount": 10, "hs_product_id": 646778218, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": null, "hs_recurring_billing_number_of_payments": 1, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 10.0, "hs_term_in_months": null, "hs_total_discount": 0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Blue Hat", "price": 10, "quantity": 1, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:11:54.030Z", "updatedAt": "2021-07-17T23:50:32.502Z", "archived": false, "properties_amount": 10.0, "properties_createdate": "2021-02-23T20:11:54.030000+00:00", "properties_description": "Baseball hat, medium", "properties_discount": null, "properties_hs_acv": 10.0, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_allow_buyer_selected_quantity": null, "properties_hs_arr": 0.0, "properties_hs_billing_period_end_date": null, "properties_hs_billing_period_start_date": null, "properties_hs_billing_start_delay_days": null, "properties_hs_billing_start_delay_months": null, "properties_hs_billing_start_delay_type": null, "properties_hs_cost_of_goods_sold": 5, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_external_id": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-07-17T23:50:32.502000+00:00", "properties_hs_line_item_currency_code": null, "properties_hs_margin": 5.0, "properties_hs_margin_acv": 5.0, "properties_hs_margin_arr": 0.0, "properties_hs_margin_mrr": 0.0, "properties_hs_margin_tcv": 5.0, "properties_hs_merged_object_ids": null, "properties_hs_mrr": 0.0, "properties_hs_object_id": 1188257165, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_position_on_quote": 0, "properties_hs_pre_discount_amount": 10, "properties_hs_product_id": 646778218, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_end_date": null, "properties_hs_recurring_billing_number_of_payments": 1, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_recurring_billing_terms": null, "properties_hs_sku": null, "properties_hs_sync_amount": null, "properties_hs_tcv": 10.0, "properties_hs_term_in_months": null, "properties_hs_total_discount": 0, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_variant_id": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Blue Hat", "properties_price": 10, "properties_quantity": 1, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1697714248811} -{"stream": "line_items", "data": {"id": "1188257309", "properties": {"amount": 10.0, "createdate": "2021-02-23T20:11:54.030000+00:00", "description": "Baseball hat, medium", "discount": null, "hs_acv": 10.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 0.0, "hs_billing_period_end_date": null, "hs_billing_period_start_date": null, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": 5, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2021-07-19T03:57:09.834000+00:00", "hs_line_item_currency_code": null, "hs_margin": 5.0, "hs_margin_acv": 5.0, "hs_margin_arr": 0.0, "hs_margin_mrr": 0.0, "hs_margin_tcv": 5.0, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_object_id": 1188257309, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_position_on_quote": 0, "hs_pre_discount_amount": 10, "hs_product_id": 646778218, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": null, "hs_recurring_billing_number_of_payments": 1, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 10.0, "hs_term_in_months": null, "hs_total_discount": 0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Blue Hat", "price": 10, "quantity": 1, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:11:54.030Z", "updatedAt": "2021-07-19T03:57:09.834Z", "archived": false, "properties_amount": 10.0, "properties_createdate": "2021-02-23T20:11:54.030000+00:00", "properties_description": "Baseball hat, medium", "properties_discount": null, "properties_hs_acv": 10.0, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_allow_buyer_selected_quantity": null, "properties_hs_arr": 0.0, "properties_hs_billing_period_end_date": null, "properties_hs_billing_period_start_date": null, "properties_hs_billing_start_delay_days": null, "properties_hs_billing_start_delay_months": null, "properties_hs_billing_start_delay_type": null, "properties_hs_cost_of_goods_sold": 5, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_external_id": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-07-19T03:57:09.834000+00:00", "properties_hs_line_item_currency_code": null, "properties_hs_margin": 5.0, "properties_hs_margin_acv": 5.0, "properties_hs_margin_arr": 0.0, "properties_hs_margin_mrr": 0.0, "properties_hs_margin_tcv": 5.0, "properties_hs_merged_object_ids": null, "properties_hs_mrr": 0.0, "properties_hs_object_id": 1188257309, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_position_on_quote": 0, "properties_hs_pre_discount_amount": 10, "properties_hs_product_id": 646778218, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_end_date": null, "properties_hs_recurring_billing_number_of_payments": 1, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_recurring_billing_terms": null, "properties_hs_sku": null, "properties_hs_sync_amount": null, "properties_hs_tcv": 10.0, "properties_hs_term_in_months": null, "properties_hs_total_discount": 0, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_variant_id": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Blue Hat", "properties_price": 10, "properties_quantity": 1, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1697714248814} -{"stream": "line_items", "data": {"id": "1510167477", "properties": {"amount": 20.0, "createdate": "2021-05-21T10:22:40.683000+00:00", "description": "Top hat, large", "discount": null, "hs_acv": 60.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 60.0, "hs_billing_period_end_date": null, "hs_billing_period_start_date": null, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": 10, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2022-02-23T08:09:16.555000+00:00", "hs_line_item_currency_code": null, "hs_margin": 10.0, "hs_margin_acv": 30.0, "hs_margin_arr": 30.0, "hs_margin_mrr": 10.0, "hs_margin_tcv": 30.0, "hs_merged_object_ids": null, "hs_mrr": 20.0, "hs_object_id": 1510167477, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_position_on_quote": null, "hs_pre_discount_amount": 20, "hs_product_id": 646777910, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": "2022-05-28", "hs_recurring_billing_number_of_payments": 3, "hs_recurring_billing_period": "P3M", "hs_recurring_billing_start_date": "2022-02-28", "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 60.0, "hs_term_in_months": 3, "hs_total_discount": 0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Red Hat", "price": 20, "quantity": 1, "recurringbillingfrequency": "monthly", "tax": null, "test": "2022-02-24", "test_product_price": "2022-02-23"}, "createdAt": "2021-05-21T10:22:40.683Z", "updatedAt": "2022-02-23T08:09:16.555Z", "archived": false, "properties_amount": 20.0, "properties_createdate": "2021-05-21T10:22:40.683000+00:00", "properties_description": "Top hat, large", "properties_discount": null, "properties_hs_acv": 60.0, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_allow_buyer_selected_quantity": null, "properties_hs_arr": 60.0, "properties_hs_billing_period_end_date": null, "properties_hs_billing_period_start_date": null, "properties_hs_billing_start_delay_days": null, "properties_hs_billing_start_delay_months": null, "properties_hs_billing_start_delay_type": null, "properties_hs_cost_of_goods_sold": 10, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_external_id": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2022-02-23T08:09:16.555000+00:00", "properties_hs_line_item_currency_code": null, "properties_hs_margin": 10.0, "properties_hs_margin_acv": 30.0, "properties_hs_margin_arr": 30.0, "properties_hs_margin_mrr": 10.0, "properties_hs_margin_tcv": 30.0, "properties_hs_merged_object_ids": null, "properties_hs_mrr": 20.0, "properties_hs_object_id": 1510167477, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_position_on_quote": null, "properties_hs_pre_discount_amount": 20, "properties_hs_product_id": 646777910, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_end_date": "2022-05-28", "properties_hs_recurring_billing_number_of_payments": 3, "properties_hs_recurring_billing_period": "P3M", "properties_hs_recurring_billing_start_date": "2022-02-28", "properties_hs_recurring_billing_terms": null, "properties_hs_sku": null, "properties_hs_sync_amount": null, "properties_hs_tcv": 60.0, "properties_hs_term_in_months": 3, "properties_hs_total_discount": 0, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_variant_id": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Red Hat", "properties_price": 20, "properties_quantity": 1, "properties_recurringbillingfrequency": "monthly", "properties_tax": null, "properties_test": "2022-02-24", "properties_test_product_price": "2022-02-23"}, "emitted_at": 1697714248816} -{"stream": "marketing_emails", "data": {"ab": false, "abHoursToWait": 4, "abSampleSizeDefault": null, "abSamplingDefault": null, "abSuccessMetric": null, "abTestPercentage": 50, "abVariation": false, "absoluteUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "aifeatures": null, "allEmailCampaignIds": [243851494], "analyticsPageId": "100523515217", "analyticsPageType": "email", "archivedAt": 0, "archivedInDashboard": false, "audienceAccess": "PUBLIC", "author": "integration-test@airbyte.io", "authorName": "Team-1 Airbyte", "blogRssSettings": null, "canSpamSettingsId": 36765207029, "categoryId": 2, "contentAccessRuleIds": [], "contentAccessRuleTypes": [], "contentTypeCategory": 2, "createPage": false, "created": 1675121582718, "createdById": 12282590, "currentState": "PUBLISHED", "currentlyPublished": true, "customReplyTo": "", "customReplyToEnabled": false, "domain": "", "emailBody": "{% content_attribute \"email_body\" %}{{ default_email_body }}{% end_content_attribute %}", "emailNote": "", "emailTemplateMode": "DRAG_AND_DROP", "emailType": "BATCH_EMAIL", "emailbodyPlaintext": "", "feedbackSurveyId": null, "flexAreas": {"main": {"boxed": false, "isSingleColumnFullWidth": false, "sections": [{"columns": [{"id": "column-0-0", "widgets": ["module-0-0-0"], "width": 12}], "id": "section-0", "style": {"backgroundColor": "#eaf0f6", "backgroundType": "CONTENT", "paddingBottom": "10px", "paddingTop": "10px"}}, {"columns": [{"id": "column-1-0", "widgets": ["module-1-0-0"], "width": 12}], "id": "section-1", "style": {"backgroundType": "CONTENT", "paddingBottom": "30px", "paddingTop": "30px"}}, {"columns": [{"id": "column-2-0", "widgets": ["module-2-0-0"], "width": 12}], "id": "section-2", "style": {"backgroundColor": "", "backgroundType": "CONTENT", "paddingBottom": "20px", "paddingTop": "20px"}}]}}, "freezeDate": 1675121645993, "fromName": "Team Airbyte", "hasContentAccessRules": false, "htmlTitle": "", "id": 100523515217, "isCreatedFomSandboxSync": false, "isGraymailSuppressionEnabled": true, "isInstanceLayoutPage": false, "isPublished": true, "isRecipientFatigueSuppressionEnabled": null, "language": "en", "layoutSections": {}, "liveDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "mailingListsExcluded": [], "mailingListsIncluded": [], "maxRssEntries": 5, "metaDescription": "", "name": "test", "pageExpiryEnabled": false, "pageRedirected": false, "pastMabExperimentIds": [], "portalId": 8727216, "previewKey": "nlkwziGL", "primaryEmailCampaignId": 243851494, "processingStatus": "PUBLISHED", "publishDate": 1675121645997, "publishImmediately": true, "publishedAt": 1675121646297, "publishedByEmail": "integration-test@airbyte.io", "publishedById": 12282590, "publishedByName": "Team-1 Airbyte", "publishedUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "replyTo": "integration-test@airbyte.io", "resolvedDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "rssEmailByText": "By", "rssEmailClickThroughText": "Read more »", "rssEmailCommentText": "Comment »", "rssEmailEntryTemplateEnabled": false, "rssEmailImageMaxWidth": 0, "rssEmailUrl": "", "sections": {}, "securityState": "NONE", "selected": 0, "slug": "-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "smartEmailFields": {}, "state": "PUBLISHED", "stats": {"counters": {"sent": 0, "open": 0, "delivered": 0, "bounce": 0, "unsubscribed": 0, "click": 0, "reply": 0, "dropped": 1, "selected": 1, "spamreport": 0, "suppressed": 0, "hardbounced": 0, "softbounced": 0, "pending": 0, "contactslost": 0, "notsent": 1}, "deviceBreakdown": {"open_device_type": {"computer": 0, "mobile": 0, "unknown": 0}, "click_device_type": {"computer": 0, "mobile": 0, "unknown": 0}}, "failedToLoad": false, "qualifierStats": {}, "ratios": {"clickratio": 0, "clickthroughratio": 0, "deliveredratio": 0, "openratio": 0, "replyratio": 0, "unsubscribedratio": 0, "spamreportratio": 0, "bounceratio": 0, "hardbounceratio": 0, "softbounceratio": 0, "contactslostratio": 0, "pendingratio": 0, "notsentratio": 100.0}}, "styleSettings": {"background_color": "#EAF0F6", "background_image": null, "background_image_type": null, "body_border_color": "#EAF0F6", "body_border_color_choice": "BORDER_MANUAL", "body_border_width": "1", "body_color": "#ffffff", "color_picker_favorite1": null, "color_picker_favorite2": null, "color_picker_favorite3": null, "color_picker_favorite4": null, "color_picker_favorite5": null, "color_picker_favorite6": null, "email_body_padding": null, "email_body_width": null, "heading_one_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "28", "underline": null}, "heading_two_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "22", "underline": null}, "links_font": {"bold": false, "color": "#00a4bd", "font": null, "font_style": {}, "italic": false, "size": null, "underline": true}, "primary_accent_color": null, "primary_font": "Arial, sans-serif", "primary_font_color": "#23496d", "primary_font_line_height": null, "primary_font_size": "15", "secondary_accent_color": null, "secondary_font": "Arial, sans-serif", "secondary_font_color": "#23496d", "secondary_font_line_height": null, "secondary_font_size": "12", "use_email_client_default_settings": false, "user_module_defaults": {"button_email": {"background_color": "#00a4bd", "corner_radius": 8, "font": "Arial, sans-serif", "font_color": "#ffffff", "font_size": 16, "font_style": {"color": "#ffffff", "font": "Arial, sans-serif", "size": {"units": "px", "value": 16}, "styles": {"bold": false, "italic": false, "underline": false}}}, "email_divider": {"color": {"color": "#23496d", "opacity": 100}, "height": 1, "line_type": "solid"}}}, "subcategory": "batch", "subject": "test", "subscription": 23704464, "subscriptionName": "Test sub", "teamPerms": [], "templatePath": "@hubspot/email/dnd/welcome.html", "transactional": false, "translations": {}, "unpublishedAt": 0, "updated": 1675121702583, "updatedById": 12282590, "url": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "useRssHeadlineAsSubject": false, "userPerms": [], "vidsExcluded": [], "vidsIncluded": [2501], "visibleToAll": true}, "emitted_at": 1697714249852} -{"stream": "marketing_emails", "data": {"ab": false, "abHoursToWait": 4, "abSampleSizeDefault": null, "abSamplingDefault": null, "abSuccessMetric": null, "abTestPercentage": 50, "abVariation": false, "absoluteUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-f142cfbc-0d58-4eb5-b442-0d221f27b420", "aifeatures": null, "allEmailCampaignIds": [169919555], "analyticsPageId": "57347028995", "analyticsPageType": "email", "archivedAt": 0, "archivedInDashboard": false, "audienceAccess": "PUBLIC", "author": "integration-test@airbyte.io", "authorName": "Team-1 Airbyte", "blogRssSettings": null, "canSpamSettingsId": 36765207029, "categoryId": 2, "contentAccessRuleIds": [], "contentAccessRuleTypes": [], "contentTypeCategory": 2, "createPage": false, "created": 1634050240841, "createdById": 12282590, "currentState": "PUBLISHED", "currentlyPublished": true, "customReplyTo": "", "customReplyToEnabled": false, "domain": "", "emailBody": "{% content_attribute \"email_body\" %}{{ default_email_body }}{% end_content_attribute %}", "emailNote": "", "emailTemplateMode": "DRAG_AND_DROP", "emailType": "BATCH_EMAIL", "emailbodyPlaintext": "", "feedbackSurveyId": null, "flexAreas": {"main": {"boxed": false, "isSingleColumnFullWidth": false, "sections": [{"columns": [{"id": "column-0-0", "widgets": ["module-0-0-0"], "width": 12}], "id": "section-0", "style": {"backgroundType": "CONTENT", "paddingBottom": "40px", "paddingTop": "40px"}}, {"columns": [{"id": "column-1-0", "widgets": ["module-1-0-0"], "width": 12}], "id": "section-1", "style": {"backgroundColor": "", "backgroundType": "CONTENT", "paddingBottom": "0px", "paddingTop": "0px"}}]}}, "freezeDate": 1634050421336, "fromName": "Team Airbyte", "hasContentAccessRules": false, "htmlTitle": "", "id": 57347028995, "isCreatedFomSandboxSync": false, "isGraymailSuppressionEnabled": true, "isInstanceLayoutPage": false, "isPublished": true, "isRecipientFatigueSuppressionEnabled": null, "language": "en", "layoutSections": {}, "liveDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "mailingListsExcluded": [], "mailingListsIncluded": [130, 129, 131, 128, 126, 127, 125, 124, 123, 122, 121, 120, 119, 118, 117, 116], "maxRssEntries": 5, "metaDescription": "", "name": "First test email - 1", "pageExpiryEnabled": false, "pageRedirected": false, "pastMabExperimentIds": [], "portalId": 8727216, "previewKey": "bgNuSvDn", "primaryEmailCampaignId": 169919555, "processingStatus": "PUBLISHED", "publishDate": 1634050421341, "publishImmediately": true, "publishedAt": 1634050421580, "publishedByEmail": "integration-test@airbyte.io", "publishedById": 12282590, "publishedByName": "Team-1 Airbyte", "publishedUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-f142cfbc-0d58-4eb5-b442-0d221f27b420", "replyTo": "integration-test@airbyte.io", "resolvedDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "rssEmailByText": "By", "rssEmailClickThroughText": "Read more »", "rssEmailCommentText": "Comment »", "rssEmailEntryTemplateEnabled": false, "rssEmailImageMaxWidth": 0, "rssEmailUrl": "", "sections": {}, "securityState": "NONE", "selected": 0, "slug": "-temporary-slug-f142cfbc-0d58-4eb5-b442-0d221f27b420", "smartEmailFields": {}, "state": "PUBLISHED", "stats": {"counters": {"sent": 0}, "deviceBreakdown": {}, "failedToLoad": false, "qualifierStats": {}, "ratios": {"clickratio": 0, "clickthroughratio": 0, "deliveredratio": 0, "openratio": 0, "replyratio": 0, "unsubscribedratio": 0, "spamreportratio": 0, "bounceratio": 0, "hardbounceratio": 0, "softbounceratio": 0, "contactslostratio": 0, "pendingratio": 0, "notsentratio": 0}}, "styleSettings": {"background_color": "#ffffff", "background_image": null, "background_image_type": null, "body_border_color": null, "body_border_color_choice": null, "body_border_width": "1", "body_color": "#ffffff", "color_picker_favorite1": null, "color_picker_favorite2": null, "color_picker_favorite3": null, "color_picker_favorite4": null, "color_picker_favorite5": null, "color_picker_favorite6": null, "email_body_padding": null, "email_body_width": null, "heading_one_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "28", "underline": null}, "heading_two_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "22", "underline": null}, "links_font": {"bold": false, "color": "#00a4bd", "font": null, "font_style": {}, "italic": false, "size": null, "underline": true}, "primary_accent_color": null, "primary_font": "Arial, sans-serif", "primary_font_color": "#23496d", "primary_font_line_height": null, "primary_font_size": "15", "secondary_accent_color": null, "secondary_font": "Arial, sans-serif", "secondary_font_color": "#23496d", "secondary_font_line_height": null, "secondary_font_size": "12", "use_email_client_default_settings": false, "user_module_defaults": {"button_email": {"background_color": null, "corner_radius": 8, "font": "Arial, sans-serif", "font_color": "#ffffff", "font_size": 16, "font_style": {"color": "#ffffff", "font": "Arial, sans-serif", "size": {"units": "px", "value": 16}, "styles": {"bold": false, "italic": false, "underline": false}}}, "email_divider": {"color": {"color": "#000000", "opacity": 100}, "height": 1, "line_type": null}}}, "subcategory": "batch", "subject": "Subject l", "subscription": 23704464, "subscriptionName": "Test sub", "teamPerms": [], "templatePath": "@hubspot/email/dnd/plain_text.html", "transactional": false, "translations": {}, "unpublishedAt": 0, "updated": 1634050455543, "updatedById": 12282590, "url": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-f142cfbc-0d58-4eb5-b442-0d221f27b420", "useRssHeadlineAsSubject": false, "userPerms": [], "vidsExcluded": [], "vidsIncluded": [], "visibleToAll": true}, "emitted_at": 1697714249853} -{"stream": "marketing_emails", "data": {"ab": false, "abHoursToWait": 4, "abSampleSizeDefault": null, "abSamplingDefault": null, "abSuccessMetric": null, "abTestPercentage": 50, "abVariation": false, "absoluteUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-fb53d6bf-1eb6-4ee6-90fe-610fc2569ea7", "aifeatures": null, "allEmailCampaignIds": [], "analyticsPageId": "42930862366", "analyticsPageType": "email", "archivedAt": 0, "archivedInDashboard": false, "audienceAccess": "PUBLIC", "author": "integration-test@airbyte.io", "authorName": "Team-1 Airbyte", "blogRssSettings": null, "canSpamSettingsId": 36765207029, "categoryId": 2, "clonedFrom": 41886608509, "contentAccessRuleIds": [], "contentAccessRuleTypes": [], "contentTypeCategory": 2, "createPage": false, "created": 1615502115346, "createdById": 100, "currentState": "AUTOMATED_DRAFT", "currentlyPublished": false, "customReplyTo": "", "customReplyToEnabled": false, "domain": "", "emailBody": "{% content_attribute \"email_body\" %}{{ default_email_body }}{% end_content_attribute %}", "emailNote": "", "emailTemplateMode": "DRAG_AND_DROP", "emailType": "AUTOMATED_EMAIL", "emailbodyPlaintext": "", "feedbackSurveyId": null, "flexAreas": {"main": {"boxed": false, "isSingleColumnFullWidth": false, "sections": [{"columns": [{"id": "column-0-1", "widgets": ["module-0-1-1"], "width": 12}], "id": "section-0", "style": {"backgroundColor": "#eaf0f6", "backgroundType": "CONTENT", "paddingBottom": "10px", "paddingTop": "10px"}}, {"columns": [{"id": "column-1-1", "widgets": ["module-1-1-1"], "width": 12}], "id": "section-1", "style": {"backgroundType": "CONTENT", "paddingBottom": "30px", "paddingTop": "30px"}}, {"columns": [{"id": "column-2-1", "widgets": ["module-2-1-1"], "width": 12}], "id": "section-2", "style": {"backgroundColor": "", "backgroundType": "CONTENT", "paddingBottom": "20px", "paddingTop": "20px"}}]}}, "freezeDate": 1634042970319, "fromName": "Team Airbyte", "hasContentAccessRules": false, "htmlTitle": "", "id": 42930862366, "isCreatedFomSandboxSync": false, "isGraymailSuppressionEnabled": false, "isInstanceLayoutPage": false, "isPublished": false, "isRecipientFatigueSuppressionEnabled": null, "language": "en", "lastEditSessionId": 1634042969643, "lastEditUpdateId": 0, "layoutSections": {}, "liveDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "mailingListsExcluded": [], "mailingListsIncluded": [], "maxRssEntries": 5, "metaDescription": "", "name": "Test subject (Test campaing - Clone)", "pageExpiryEnabled": false, "pageRedirected": false, "pastMabExperimentIds": [], "portalId": 8727216, "previewKey": "UmZGYZsU", "processingStatus": "UNDEFINED", "publishDate": 1634042970320, "publishImmediately": true, "publishedUrl": "", "replyTo": "integration-test@airbyte.io", "resolvedDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "rssEmailByText": "By", "rssEmailClickThroughText": "Read more »", "rssEmailCommentText": "Comment »", "rssEmailEntryTemplateEnabled": false, "rssEmailImageMaxWidth": 0, "rssEmailUrl": "", "sections": {}, "securityState": "NONE", "slug": "-temporary-slug-fb53d6bf-1eb6-4ee6-90fe-610fc2569ea7", "smartEmailFields": {}, "state": "AUTOMATED_DRAFT", "styleSettings": {"background_color": "#EAF0F6", "background_image": null, "background_image_type": null, "body_border_color": "#EAF0F6", "body_border_color_choice": "BORDER_MANUAL", "body_border_width": "1", "body_color": "#ffffff", "color_picker_favorite1": null, "color_picker_favorite2": null, "color_picker_favorite3": null, "color_picker_favorite4": null, "color_picker_favorite5": null, "color_picker_favorite6": null, "email_body_padding": null, "email_body_width": null, "heading_one_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "28", "underline": null}, "heading_two_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "22", "underline": null}, "links_font": {"bold": false, "color": "#00a4bd", "font": null, "font_style": {}, "italic": false, "size": null, "underline": true}, "primary_accent_color": null, "primary_font": "Arial, sans-serif", "primary_font_color": "#23496d", "primary_font_line_height": null, "primary_font_size": "15", "secondary_accent_color": null, "secondary_font": "Arial, sans-serif", "secondary_font_color": "#23496d", "secondary_font_line_height": null, "secondary_font_size": "12", "use_email_client_default_settings": false, "user_module_defaults": {"button_email": {"background_color": "#00a4bd", "corner_radius": 8, "font": "Arial, sans-serif", "font_color": "#ffffff", "font_size": 16, "font_style": {"color": "#ffffff", "font": "Arial, sans-serif", "size": {"units": "px", "value": 16}, "styles": {"bold": false, "italic": false, "underline": false}}}, "email_divider": {"color": {"color": "#23496d", "opacity": 100}, "height": 1, "line_type": "solid"}}}, "subcategory": "automated", "subject": "Test subject", "subscription": 11890831, "subscriptionName": "Test subscription", "teamPerms": [], "templatePath": "@hubspot/email/dnd/welcome.html", "transactional": false, "translations": {}, "unpublishedAt": 0, "updated": 1634042970321, "updatedById": 12282590, "url": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-fb53d6bf-1eb6-4ee6-90fe-610fc2569ea7", "useRssHeadlineAsSubject": false, "userPerms": [], "vidsExcluded": [], "vidsIncluded": [], "visibleToAll": true}, "emitted_at": 1697714249854} +{"stream": "goals", "data": {"id": "221880757009", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-07-31T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_definition_key": null, "hs_goal_definition_key_with_team": null, "hs_goal_definition_key_with_user": null, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_group_without_team_id": null, "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "hs_kpi_is_team_rollup": false, "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-08-01T00:45:14.830000+00:00", "hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757009, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_outcome": "completed", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-07-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "achieved", "hs_status_display_order": 4, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_template_id": 4, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-12-11T20:46:14.473Z", "archived": false, "properties_hs__migration_soft_delete": null, "properties_hs_ad_account_asset_ids": null, "properties_hs_ad_campaign_asset_ids": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignee_team_id": null, "properties_hs_assignee_user_id": 26748728, "properties_hs_contact_lifecycle_stage": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-10T13:57:36.691000+00:00", "properties_hs_currency": null, "properties_hs_deal_pipeline_ids": null, "properties_hs_edit_updates_notification_frequency": "weekly", "properties_hs_end_date": null, "properties_hs_end_datetime": "2023-07-31T23:59:59.999000+00:00", "properties_hs_fiscal_year_offset": 0, "properties_hs_goal_definition_key": null, "properties_hs_goal_definition_key_with_team": null, "properties_hs_goal_definition_key_with_user": null, "properties_hs_goal_name": "Integration Test Goal Hubspot", "properties_hs_goal_target_group_id": 221880750627, "properties_hs_goal_type": "average_ticket_response_time", "properties_hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "properties_hs_is_forecastable": "true", "properties_hs_is_legacy": null, "properties_hs_kpi_display_unit": "hour", "properties_hs_kpi_filter_group_without_team_id": null, "properties_hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "properties_hs_kpi_is_team_rollup": false, "properties_hs_kpi_metric_type": "AVG", "properties_hs_kpi_object_type": "TICKET", "properties_hs_kpi_object_type_id": "0-5", "properties_hs_kpi_progress_percent": null, "properties_hs_kpi_property_name": "time_to_first_agent_reply", "properties_hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "properties_hs_kpi_time_period_property": "createdate", "properties_hs_kpi_tracking_method": "LOWER_IS_BETTER", "properties_hs_kpi_unit_type": "duration", "properties_hs_kpi_value": 0.0, "properties_hs_kpi_value_calculated_at": null, "properties_hs_kpi_value_last_calculated_at": "2023-08-01T00:45:14.830000+00:00", "properties_hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "properties_hs_legacy_active": null, "properties_hs_legacy_created_at": null, "properties_hs_legacy_created_by": null, "properties_hs_legacy_quarterly_target_composite_id": null, "properties_hs_legacy_sql_id": null, "properties_hs_legacy_unique_sql_id": null, "properties_hs_legacy_updated_at": null, "properties_hs_legacy_updated_by": null, "properties_hs_merged_object_ids": null, "properties_hs_migration_soft_delete": null, "properties_hs_milestone": "monthly", "properties_hs_object_id": 221880757009, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_outcome": "completed", "properties_hs_owner_ids_of_all_owners": "111730024", "properties_hs_participant_type": "users", "properties_hs_pipelines": "0", "properties_hs_progress_updates_notification_frequency": "weekly", "properties_hs_read_only": null, "properties_hs_should_notify_on_achieved": "false", "properties_hs_should_notify_on_edit_updates": "false", "properties_hs_should_notify_on_exceeded": "false", "properties_hs_should_notify_on_kickoff": "false", "properties_hs_should_notify_on_missed": "false", "properties_hs_should_notify_on_progress_updates": "false", "properties_hs_should_recalculate": "false", "properties_hs_start_date": null, "properties_hs_start_datetime": "2023-07-01T00:00:00+00:00", "properties_hs_static_kpi_filter_groups": "[]", "properties_hs_status": "achieved", "properties_hs_status_display_order": 4, "properties_hs_target_amount": 0.0, "properties_hs_target_amount_in_home_currency": 0.0, "properties_hs_team_id": null, "properties_hs_template_id": 4, "properties_hs_ticket_pipeline_ids": "0", "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "26748728", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null}, "emitted_at": 1706192378510} +{"stream": "goals", "data": {"id": "221880757010", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-09-30T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_definition_key": null, "hs_goal_definition_key_with_team": null, "hs_goal_definition_key_with_user": null, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_group_without_team_id": null, "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "hs_kpi_is_team_rollup": false, "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-10-01T22:31:08.621000+00:00", "hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757010, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_outcome": "completed", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-09-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "achieved", "hs_status_display_order": 4, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_template_id": 4, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-12-11T20:46:14.473Z", "archived": false, "properties_hs__migration_soft_delete": null, "properties_hs_ad_account_asset_ids": null, "properties_hs_ad_campaign_asset_ids": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignee_team_id": null, "properties_hs_assignee_user_id": 26748728, "properties_hs_contact_lifecycle_stage": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-10T13:57:36.691000+00:00", "properties_hs_currency": null, "properties_hs_deal_pipeline_ids": null, "properties_hs_edit_updates_notification_frequency": "weekly", "properties_hs_end_date": null, "properties_hs_end_datetime": "2023-09-30T23:59:59.999000+00:00", "properties_hs_fiscal_year_offset": 0, "properties_hs_goal_definition_key": null, "properties_hs_goal_definition_key_with_team": null, "properties_hs_goal_definition_key_with_user": null, "properties_hs_goal_name": "Integration Test Goal Hubspot", "properties_hs_goal_target_group_id": 221880750627, "properties_hs_goal_type": "average_ticket_response_time", "properties_hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "properties_hs_is_forecastable": "true", "properties_hs_is_legacy": null, "properties_hs_kpi_display_unit": "hour", "properties_hs_kpi_filter_group_without_team_id": null, "properties_hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "properties_hs_kpi_is_team_rollup": false, "properties_hs_kpi_metric_type": "AVG", "properties_hs_kpi_object_type": "TICKET", "properties_hs_kpi_object_type_id": "0-5", "properties_hs_kpi_progress_percent": null, "properties_hs_kpi_property_name": "time_to_first_agent_reply", "properties_hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "properties_hs_kpi_time_period_property": "createdate", "properties_hs_kpi_tracking_method": "LOWER_IS_BETTER", "properties_hs_kpi_unit_type": "duration", "properties_hs_kpi_value": 0.0, "properties_hs_kpi_value_calculated_at": null, "properties_hs_kpi_value_last_calculated_at": "2023-10-01T22:31:08.621000+00:00", "properties_hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "properties_hs_legacy_active": null, "properties_hs_legacy_created_at": null, "properties_hs_legacy_created_by": null, "properties_hs_legacy_quarterly_target_composite_id": null, "properties_hs_legacy_sql_id": null, "properties_hs_legacy_unique_sql_id": null, "properties_hs_legacy_updated_at": null, "properties_hs_legacy_updated_by": null, "properties_hs_merged_object_ids": null, "properties_hs_migration_soft_delete": null, "properties_hs_milestone": "monthly", "properties_hs_object_id": 221880757010, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_outcome": "completed", "properties_hs_owner_ids_of_all_owners": "111730024", "properties_hs_participant_type": "users", "properties_hs_pipelines": "0", "properties_hs_progress_updates_notification_frequency": "weekly", "properties_hs_read_only": null, "properties_hs_should_notify_on_achieved": "false", "properties_hs_should_notify_on_edit_updates": "false", "properties_hs_should_notify_on_exceeded": "false", "properties_hs_should_notify_on_kickoff": "false", "properties_hs_should_notify_on_missed": "false", "properties_hs_should_notify_on_progress_updates": "false", "properties_hs_should_recalculate": "false", "properties_hs_start_date": null, "properties_hs_start_datetime": "2023-09-01T00:00:00+00:00", "properties_hs_static_kpi_filter_groups": "[]", "properties_hs_status": "achieved", "properties_hs_status_display_order": 4, "properties_hs_target_amount": 0.0, "properties_hs_target_amount_in_home_currency": 0.0, "properties_hs_team_id": null, "properties_hs_template_id": 4, "properties_hs_ticket_pipeline_ids": "0", "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "26748728", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null}, "emitted_at": 1706192378516} +{"stream": "goals", "data": {"id": "221880757011", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-08-31T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_definition_key": null, "hs_goal_definition_key_with_team": null, "hs_goal_definition_key_with_user": null, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_group_without_team_id": null, "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "hs_kpi_is_team_rollup": false, "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-09-01T15:26:00.500000+00:00", "hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757011, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_outcome": "completed", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-08-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "achieved", "hs_status_display_order": 4, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_template_id": 4, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-12-11T20:46:14.473Z", "archived": false, "properties_hs__migration_soft_delete": null, "properties_hs_ad_account_asset_ids": null, "properties_hs_ad_campaign_asset_ids": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignee_team_id": null, "properties_hs_assignee_user_id": 26748728, "properties_hs_contact_lifecycle_stage": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-10T13:57:36.691000+00:00", "properties_hs_currency": null, "properties_hs_deal_pipeline_ids": null, "properties_hs_edit_updates_notification_frequency": "weekly", "properties_hs_end_date": null, "properties_hs_end_datetime": "2023-08-31T23:59:59.999000+00:00", "properties_hs_fiscal_year_offset": 0, "properties_hs_goal_definition_key": null, "properties_hs_goal_definition_key_with_team": null, "properties_hs_goal_definition_key_with_user": null, "properties_hs_goal_name": "Integration Test Goal Hubspot", "properties_hs_goal_target_group_id": 221880750627, "properties_hs_goal_type": "average_ticket_response_time", "properties_hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "properties_hs_is_forecastable": "true", "properties_hs_is_legacy": null, "properties_hs_kpi_display_unit": "hour", "properties_hs_kpi_filter_group_without_team_id": null, "properties_hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "properties_hs_kpi_is_team_rollup": false, "properties_hs_kpi_metric_type": "AVG", "properties_hs_kpi_object_type": "TICKET", "properties_hs_kpi_object_type_id": "0-5", "properties_hs_kpi_progress_percent": null, "properties_hs_kpi_property_name": "time_to_first_agent_reply", "properties_hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "properties_hs_kpi_time_period_property": "createdate", "properties_hs_kpi_tracking_method": "LOWER_IS_BETTER", "properties_hs_kpi_unit_type": "duration", "properties_hs_kpi_value": 0.0, "properties_hs_kpi_value_calculated_at": null, "properties_hs_kpi_value_last_calculated_at": "2023-09-01T15:26:00.500000+00:00", "properties_hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "properties_hs_legacy_active": null, "properties_hs_legacy_created_at": null, "properties_hs_legacy_created_by": null, "properties_hs_legacy_quarterly_target_composite_id": null, "properties_hs_legacy_sql_id": null, "properties_hs_legacy_unique_sql_id": null, "properties_hs_legacy_updated_at": null, "properties_hs_legacy_updated_by": null, "properties_hs_merged_object_ids": null, "properties_hs_migration_soft_delete": null, "properties_hs_milestone": "monthly", "properties_hs_object_id": 221880757011, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_outcome": "completed", "properties_hs_owner_ids_of_all_owners": "111730024", "properties_hs_participant_type": "users", "properties_hs_pipelines": "0", "properties_hs_progress_updates_notification_frequency": "weekly", "properties_hs_read_only": null, "properties_hs_should_notify_on_achieved": "false", "properties_hs_should_notify_on_edit_updates": "false", "properties_hs_should_notify_on_exceeded": "false", "properties_hs_should_notify_on_kickoff": "false", "properties_hs_should_notify_on_missed": "false", "properties_hs_should_notify_on_progress_updates": "false", "properties_hs_should_recalculate": "false", "properties_hs_start_date": null, "properties_hs_start_datetime": "2023-08-01T00:00:00+00:00", "properties_hs_static_kpi_filter_groups": "[]", "properties_hs_status": "achieved", "properties_hs_status_display_order": 4, "properties_hs_target_amount": 0.0, "properties_hs_target_amount_in_home_currency": 0.0, "properties_hs_team_id": null, "properties_hs_template_id": 4, "properties_hs_ticket_pipeline_ids": "0", "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "26748728", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null}, "emitted_at": 1706192378517} +{"stream": "line_items", "data": {"id": "1188257165", "properties": {"amount": 10.0, "createdate": "2021-02-23T20:11:54.030000+00:00", "description": "Baseball hat, medium", "discount": null, "hs_acv": 10.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 0.0, "hs_billing_period_end_date": null, "hs_billing_period_start_date": null, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": 5, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2021-07-17T23:50:32.502000+00:00", "hs_line_item_currency_code": null, "hs_margin": 5.0, "hs_margin_acv": 5.0, "hs_margin_arr": 0.0, "hs_margin_mrr": 0.0, "hs_margin_tcv": 5.0, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_object_id": 1188257165, "hs_object_source": "CRM_UI", "hs_object_source_id": "integration-test@airbyte.io", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_position_on_quote": 0, "hs_pre_discount_amount": 10, "hs_product_id": 646778218, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": null, "hs_recurring_billing_number_of_payments": 1, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 10.0, "hs_term_in_months": null, "hs_total_discount": 0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Blue Hat", "price": 10, "quantity": 1, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:11:54.030Z", "updatedAt": "2021-07-17T23:50:32.502Z", "archived": false, "properties_amount": 10.0, "properties_createdate": "2021-02-23T20:11:54.030000+00:00", "properties_description": "Baseball hat, medium", "properties_discount": null, "properties_hs_acv": 10.0, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_allow_buyer_selected_quantity": null, "properties_hs_arr": 0.0, "properties_hs_billing_period_end_date": null, "properties_hs_billing_period_start_date": null, "properties_hs_billing_start_delay_days": null, "properties_hs_billing_start_delay_months": null, "properties_hs_billing_start_delay_type": null, "properties_hs_cost_of_goods_sold": 5, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_external_id": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-07-17T23:50:32.502000+00:00", "properties_hs_line_item_currency_code": null, "properties_hs_margin": 5.0, "properties_hs_margin_acv": 5.0, "properties_hs_margin_arr": 0.0, "properties_hs_margin_mrr": 0.0, "properties_hs_margin_tcv": 5.0, "properties_hs_merged_object_ids": null, "properties_hs_mrr": 0.0, "properties_hs_object_id": 1188257165, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_id": "integration-test@airbyte.io", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_position_on_quote": 0, "properties_hs_pre_discount_amount": 10, "properties_hs_product_id": 646778218, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_end_date": null, "properties_hs_recurring_billing_number_of_payments": 1, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_recurring_billing_terms": null, "properties_hs_sku": null, "properties_hs_sync_amount": null, "properties_hs_tcv": 10.0, "properties_hs_term_in_months": null, "properties_hs_total_discount": 0, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_variant_id": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Blue Hat", "properties_price": 10, "properties_quantity": 1, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1706192514669} +{"stream": "line_items", "data": {"id": "1188257309", "properties": {"amount": 10.0, "createdate": "2021-02-23T20:11:54.030000+00:00", "description": "Baseball hat, medium", "discount": null, "hs_acv": 10.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 0.0, "hs_billing_period_end_date": null, "hs_billing_period_start_date": null, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": 5, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2021-07-19T03:57:09.834000+00:00", "hs_line_item_currency_code": null, "hs_margin": 5.0, "hs_margin_acv": 5.0, "hs_margin_arr": 0.0, "hs_margin_mrr": 0.0, "hs_margin_tcv": 5.0, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_object_id": 1188257309, "hs_object_source": "CRM_UI", "hs_object_source_id": "integration-test@airbyte.io", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_position_on_quote": 0, "hs_pre_discount_amount": 10, "hs_product_id": 646778218, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": null, "hs_recurring_billing_number_of_payments": 1, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 10.0, "hs_term_in_months": null, "hs_total_discount": 0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Blue Hat", "price": 10, "quantity": 1, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:11:54.030Z", "updatedAt": "2021-07-19T03:57:09.834Z", "archived": false, "properties_amount": 10.0, "properties_createdate": "2021-02-23T20:11:54.030000+00:00", "properties_description": "Baseball hat, medium", "properties_discount": null, "properties_hs_acv": 10.0, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_allow_buyer_selected_quantity": null, "properties_hs_arr": 0.0, "properties_hs_billing_period_end_date": null, "properties_hs_billing_period_start_date": null, "properties_hs_billing_start_delay_days": null, "properties_hs_billing_start_delay_months": null, "properties_hs_billing_start_delay_type": null, "properties_hs_cost_of_goods_sold": 5, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_external_id": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-07-19T03:57:09.834000+00:00", "properties_hs_line_item_currency_code": null, "properties_hs_margin": 5.0, "properties_hs_margin_acv": 5.0, "properties_hs_margin_arr": 0.0, "properties_hs_margin_mrr": 0.0, "properties_hs_margin_tcv": 5.0, "properties_hs_merged_object_ids": null, "properties_hs_mrr": 0.0, "properties_hs_object_id": 1188257309, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_id": "integration-test@airbyte.io", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_position_on_quote": 0, "properties_hs_pre_discount_amount": 10, "properties_hs_product_id": 646778218, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_end_date": null, "properties_hs_recurring_billing_number_of_payments": 1, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_recurring_billing_terms": null, "properties_hs_sku": null, "properties_hs_sync_amount": null, "properties_hs_tcv": 10.0, "properties_hs_term_in_months": null, "properties_hs_total_discount": 0, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_variant_id": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Blue Hat", "properties_price": 10, "properties_quantity": 1, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1706192514672} +{"stream": "line_items", "data": {"id": "1510167477", "properties": {"amount": 20.0, "createdate": "2021-05-21T10:22:40.683000+00:00", "description": "Top hat, large", "discount": null, "hs_acv": 60.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 60.0, "hs_billing_period_end_date": null, "hs_billing_period_start_date": null, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": 10, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2022-02-23T08:09:16.555000+00:00", "hs_line_item_currency_code": null, "hs_margin": 10.0, "hs_margin_acv": 30.0, "hs_margin_arr": 30.0, "hs_margin_mrr": 10.0, "hs_margin_tcv": 30.0, "hs_merged_object_ids": null, "hs_mrr": 20.0, "hs_object_id": 1510167477, "hs_object_source": "CRM_UI", "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_position_on_quote": null, "hs_pre_discount_amount": 20, "hs_product_id": 646777910, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": "2022-05-28", "hs_recurring_billing_number_of_payments": 3, "hs_recurring_billing_period": "P3M", "hs_recurring_billing_start_date": "2022-02-28", "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 60.0, "hs_term_in_months": 3, "hs_total_discount": 0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Red Hat", "price": 20, "quantity": 1, "recurringbillingfrequency": "monthly", "tax": null, "test": "2022-02-24", "test_product_price": "2022-02-23"}, "createdAt": "2021-05-21T10:22:40.683Z", "updatedAt": "2022-02-23T08:09:16.555Z", "archived": false, "properties_amount": 20.0, "properties_createdate": "2021-05-21T10:22:40.683000+00:00", "properties_description": "Top hat, large", "properties_discount": null, "properties_hs_acv": 60.0, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_allow_buyer_selected_quantity": null, "properties_hs_arr": 60.0, "properties_hs_billing_period_end_date": null, "properties_hs_billing_period_start_date": null, "properties_hs_billing_start_delay_days": null, "properties_hs_billing_start_delay_months": null, "properties_hs_billing_start_delay_type": null, "properties_hs_cost_of_goods_sold": 10, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_external_id": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2022-02-23T08:09:16.555000+00:00", "properties_hs_line_item_currency_code": null, "properties_hs_margin": 10.0, "properties_hs_margin_acv": 30.0, "properties_hs_margin_arr": 30.0, "properties_hs_margin_mrr": 10.0, "properties_hs_margin_tcv": 30.0, "properties_hs_merged_object_ids": null, "properties_hs_mrr": 20.0, "properties_hs_object_id": 1510167477, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_position_on_quote": null, "properties_hs_pre_discount_amount": 20, "properties_hs_product_id": 646777910, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_end_date": "2022-05-28", "properties_hs_recurring_billing_number_of_payments": 3, "properties_hs_recurring_billing_period": "P3M", "properties_hs_recurring_billing_start_date": "2022-02-28", "properties_hs_recurring_billing_terms": null, "properties_hs_sku": null, "properties_hs_sync_amount": null, "properties_hs_tcv": 60.0, "properties_hs_term_in_months": 3, "properties_hs_total_discount": 0, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_variant_id": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Red Hat", "properties_price": 20, "properties_quantity": 1, "properties_recurringbillingfrequency": "monthly", "properties_tax": null, "properties_test": "2022-02-24", "properties_test_product_price": "2022-02-23"}, "emitted_at": 1706192514674} +{"stream": "marketing_emails", "data": {"ab": false, "abHoursToWait": 4, "abSampleSizeDefault": null, "abSamplingDefault": null, "abSuccessMetric": null, "abTestPercentage": 50, "abVariation": false, "absoluteUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "aifeatures": null, "allEmailCampaignIds": [243851494], "analyticsPageId": "100523515217", "analyticsPageType": "email", "archivedAt": 0, "archivedInDashboard": false, "audienceAccess": "PUBLIC", "author": "integration-test@airbyte.io", "authorName": "Team-1 Airbyte", "blogRssSettings": null, "canSpamSettingsId": 36765207029, "categoryId": 2, "contentAccessRuleIds": [], "contentAccessRuleTypes": [], "contentTypeCategory": 2, "createPage": false, "created": 1675121582718, "createdById": 12282590, "currentState": "PUBLISHED", "currentlyPublished": true, "customReplyTo": "", "customReplyToEnabled": false, "domain": "", "emailBody": "{% content_attribute \"email_body\" %}{{ default_email_body }}{% end_content_attribute %}", "emailNote": "", "emailTemplateMode": "DRAG_AND_DROP", "emailType": "BATCH_EMAIL", "emailbodyPlaintext": "", "feedbackSurveyId": null, "flexAreas": {"main": {"boxed": false, "isSingleColumnFullWidth": false, "sections": [{"columns": [{"id": "column-0-0", "widgets": ["module-0-0-0"], "width": 12}], "id": "section-0", "style": {"backgroundColor": "#eaf0f6", "backgroundType": "CONTENT", "paddingBottom": "10px", "paddingTop": "10px"}}, {"columns": [{"id": "column-1-0", "widgets": ["module-1-0-0"], "width": 12}], "id": "section-1", "style": {"backgroundType": "CONTENT", "paddingBottom": "30px", "paddingTop": "30px"}}, {"columns": [{"id": "column-2-0", "widgets": ["module-2-0-0"], "width": 12}], "id": "section-2", "style": {"backgroundColor": "", "backgroundType": "CONTENT", "paddingBottom": "20px", "paddingTop": "20px"}}]}}, "freezeDate": 1675121645993, "fromName": "Team Airbyte", "hasContentAccessRules": false, "htmlTitle": "", "id": 100523515217, "isCreatedFomSandboxSync": false, "isGraymailSuppressionEnabled": true, "isInstanceLayoutPage": false, "isPublished": true, "isRecipientFatigueSuppressionEnabled": null, "language": "en", "layoutSections": {}, "liveDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "mailingListsExcluded": [], "mailingListsIncluded": [], "maxRssEntries": 5, "metaDescription": "", "name": "test", "pageExpiryEnabled": false, "pageRedirected": false, "pastMabExperimentIds": [], "portalId": 8727216, "previewKey": "nlkwziGL", "primaryEmailCampaignId": 243851494, "processingStatus": "PUBLISHED", "publishDate": 1675121645000, "publishImmediately": true, "publishedAt": 1675121646297, "publishedByEmail": "integration-test@airbyte.io", "publishedById": 12282590, "publishedByName": "Team-1 Airbyte", "publishedUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "replyTo": "integration-test@airbyte.io", "resolvedDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "rootMicId": null, "rssEmailByText": "By", "rssEmailClickThroughText": "Read more »", "rssEmailCommentText": "Comment »", "rssEmailEntryTemplateEnabled": false, "rssEmailImageMaxWidth": 0, "rssEmailUrl": "", "sections": {}, "securityState": "NONE", "selected": 0, "slug": "-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "smartEmailFields": {}, "state": "PUBLISHED", "stats": {"counters": {"sent": 0, "open": 0, "delivered": 0, "bounce": 0, "unsubscribed": 0, "click": 0, "reply": 0, "dropped": 1, "selected": 1, "spamreport": 0, "suppressed": 0, "hardbounced": 0, "softbounced": 0, "pending": 0, "contactslost": 0, "notsent": 1}, "deviceBreakdown": {"open_device_type": {"computer": 0, "mobile": 0, "unknown": 0}, "click_device_type": {"computer": 0, "mobile": 0, "unknown": 0}}, "failedToLoad": false, "qualifierStats": {}, "ratios": {"clickratio": 0, "clickthroughratio": 0, "deliveredratio": 0, "openratio": 0, "replyratio": 0, "unsubscribedratio": 0, "spamreportratio": 0, "bounceratio": 0, "hardbounceratio": 0, "softbounceratio": 0, "contactslostratio": 0, "pendingratio": 0, "notsentratio": 100.0}}, "styleSettings": {"background_color": "#EAF0F6", "background_image": null, "background_image_type": null, "body_border_color": "#EAF0F6", "body_border_color_choice": "BORDER_MANUAL", "body_border_width": "1", "body_color": "#ffffff", "color_picker_favorite1": null, "color_picker_favorite2": null, "color_picker_favorite3": null, "color_picker_favorite4": null, "color_picker_favorite5": null, "color_picker_favorite6": null, "email_body_padding": null, "email_body_width": null, "heading_one_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "28", "underline": null}, "heading_two_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "22", "underline": null}, "links_font": {"bold": false, "color": "#00a4bd", "font": null, "font_style": {}, "italic": false, "size": null, "underline": true}, "primary_accent_color": null, "primary_font": "Arial, sans-serif", "primary_font_color": "#23496d", "primary_font_line_height": null, "primary_font_size": "15", "secondary_accent_color": null, "secondary_font": "Arial, sans-serif", "secondary_font_color": "#23496d", "secondary_font_line_height": null, "secondary_font_size": "12", "use_email_client_default_settings": false, "user_module_defaults": {"button_email": {"background_color": "#00a4bd", "corner_radius": 8, "font": "Arial, sans-serif", "font_color": "#ffffff", "font_size": 16, "font_style": {"color": "#ffffff", "font": "Arial, sans-serif", "size": {"units": "px", "value": 16}, "styles": {"bold": false, "italic": false, "underline": false}}}, "email_divider": {"color": {"color": "#23496d", "opacity": 100}, "height": 1, "line_type": "solid"}}}, "subcategory": "batch", "subject": "test", "subscription": 23704464, "subscriptionName": "Test sub", "teamPerms": [], "templatePath": "@hubspot/email/dnd/welcome.html", "transactional": false, "translations": {}, "unpublishedAt": 0, "updated": 1675121702583, "updatedById": 12282590, "url": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "useRssHeadlineAsSubject": false, "userPerms": [], "vidsExcluded": [], "vidsIncluded": [2501], "visibleToAll": true}, "emitted_at": 1706192639128} +{"stream": "marketing_emails", "data": {"ab": false, "abHoursToWait": 4, "abSampleSizeDefault": null, "abSamplingDefault": null, "abSuccessMetric": null, "abTestPercentage": 50, "abVariation": false, "absoluteUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-f142cfbc-0d58-4eb5-b442-0d221f27b420", "aifeatures": null, "allEmailCampaignIds": [169919555], "analyticsPageId": "57347028995", "analyticsPageType": "email", "archivedAt": 0, "archivedInDashboard": false, "audienceAccess": "PUBLIC", "author": "integration-test@airbyte.io", "authorName": "Team-1 Airbyte", "blogRssSettings": null, "canSpamSettingsId": 36765207029, "categoryId": 2, "contentAccessRuleIds": [], "contentAccessRuleTypes": [], "contentTypeCategory": 2, "createPage": false, "created": 1634050240841, "createdById": 12282590, "currentState": "PUBLISHED", "currentlyPublished": true, "customReplyTo": "", "customReplyToEnabled": false, "domain": "", "emailBody": "{% content_attribute \"email_body\" %}{{ default_email_body }}{% end_content_attribute %}", "emailNote": "", "emailTemplateMode": "DRAG_AND_DROP", "emailType": "BATCH_EMAIL", "emailbodyPlaintext": "", "feedbackSurveyId": null, "flexAreas": {"main": {"boxed": false, "isSingleColumnFullWidth": false, "sections": [{"columns": [{"id": "column-0-0", "widgets": ["module-0-0-0"], "width": 12}], "id": "section-0", "style": {"backgroundType": "CONTENT", "paddingBottom": "40px", "paddingTop": "40px"}}, {"columns": [{"id": "column-1-0", "widgets": ["module-1-0-0"], "width": 12}], "id": "section-1", "style": {"backgroundColor": "", "backgroundType": "CONTENT", "paddingBottom": "0px", "paddingTop": "0px"}}]}}, "freezeDate": 1634050421336, "fromName": "Team Airbyte", "hasContentAccessRules": false, "htmlTitle": "", "id": 57347028995, "isCreatedFomSandboxSync": false, "isGraymailSuppressionEnabled": true, "isInstanceLayoutPage": false, "isPublished": true, "isRecipientFatigueSuppressionEnabled": null, "language": "en", "layoutSections": {}, "liveDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "mailingListsExcluded": [], "mailingListsIncluded": [130, 129, 131, 128, 126, 127, 125, 124, 123, 122, 121, 120, 119, 118, 117, 116], "maxRssEntries": 5, "metaDescription": "", "name": "First test email - 1", "pageExpiryEnabled": false, "pageRedirected": false, "pastMabExperimentIds": [], "portalId": 8727216, "previewKey": "bgNuSvDn", "primaryEmailCampaignId": 169919555, "processingStatus": "PUBLISHED", "publishDate": 1634050421000, "publishImmediately": true, "publishedAt": 1634050421580, "publishedByEmail": "integration-test@airbyte.io", "publishedById": 12282590, "publishedByName": "Team-1 Airbyte", "publishedUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-f142cfbc-0d58-4eb5-b442-0d221f27b420", "replyTo": "integration-test@airbyte.io", "resolvedDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "rootMicId": null, "rssEmailByText": "By", "rssEmailClickThroughText": "Read more »", "rssEmailCommentText": "Comment »", "rssEmailEntryTemplateEnabled": false, "rssEmailImageMaxWidth": 0, "rssEmailUrl": "", "sections": {}, "securityState": "NONE", "selected": 0, "slug": "-temporary-slug-f142cfbc-0d58-4eb5-b442-0d221f27b420", "smartEmailFields": {}, "state": "PUBLISHED", "stats": {"counters": {"sent": 0}, "deviceBreakdown": {}, "failedToLoad": false, "qualifierStats": {}, "ratios": {"clickratio": 0, "clickthroughratio": 0, "deliveredratio": 0, "openratio": 0, "replyratio": 0, "unsubscribedratio": 0, "spamreportratio": 0, "bounceratio": 0, "hardbounceratio": 0, "softbounceratio": 0, "contactslostratio": 0, "pendingratio": 0, "notsentratio": 0}}, "styleSettings": {"background_color": "#ffffff", "background_image": null, "background_image_type": null, "body_border_color": null, "body_border_color_choice": null, "body_border_width": "1", "body_color": "#ffffff", "color_picker_favorite1": null, "color_picker_favorite2": null, "color_picker_favorite3": null, "color_picker_favorite4": null, "color_picker_favorite5": null, "color_picker_favorite6": null, "email_body_padding": null, "email_body_width": null, "heading_one_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "28", "underline": null}, "heading_two_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "22", "underline": null}, "links_font": {"bold": false, "color": "#00a4bd", "font": null, "font_style": {}, "italic": false, "size": null, "underline": true}, "primary_accent_color": null, "primary_font": "Arial, sans-serif", "primary_font_color": "#23496d", "primary_font_line_height": null, "primary_font_size": "15", "secondary_accent_color": null, "secondary_font": "Arial, sans-serif", "secondary_font_color": "#23496d", "secondary_font_line_height": null, "secondary_font_size": "12", "use_email_client_default_settings": false, "user_module_defaults": {"button_email": {"background_color": null, "corner_radius": 8, "font": "Arial, sans-serif", "font_color": "#ffffff", "font_size": 16, "font_style": {"color": "#ffffff", "font": "Arial, sans-serif", "size": {"units": "px", "value": 16}, "styles": {"bold": false, "italic": false, "underline": false}}}, "email_divider": {"color": {"color": "#000000", "opacity": 100}, "height": 1, "line_type": null}}}, "subcategory": "batch", "subject": "Subject l", "subscription": 23704464, "subscriptionName": "Test sub", "teamPerms": [], "templatePath": "@hubspot/email/dnd/plain_text.html", "transactional": false, "translations": {}, "unpublishedAt": 0, "updated": 1634050455543, "updatedById": 12282590, "url": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-f142cfbc-0d58-4eb5-b442-0d221f27b420", "useRssHeadlineAsSubject": false, "userPerms": [], "vidsExcluded": [], "vidsIncluded": [], "visibleToAll": true}, "emitted_at": 1706192639130} +{"stream": "marketing_emails", "data": {"ab": false, "abHoursToWait": 4, "abSampleSizeDefault": null, "abSamplingDefault": null, "abSuccessMetric": null, "abTestPercentage": 50, "abVariation": false, "absoluteUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-fb53d6bf-1eb6-4ee6-90fe-610fc2569ea7", "aifeatures": null, "allEmailCampaignIds": [], "analyticsPageId": "42930862366", "analyticsPageType": "email", "archivedAt": 0, "archivedInDashboard": false, "audienceAccess": "PUBLIC", "author": "integration-test@airbyte.io", "authorName": "Team-1 Airbyte", "blogRssSettings": null, "canSpamSettingsId": 36765207029, "categoryId": 2, "clonedFrom": 41886608509, "contentAccessRuleIds": [], "contentAccessRuleTypes": [], "contentTypeCategory": 2, "createPage": false, "created": 1615502115346, "createdById": 100, "currentState": "AUTOMATED_DRAFT", "currentlyPublished": false, "customReplyTo": "", "customReplyToEnabled": false, "domain": "", "emailBody": "{% content_attribute \"email_body\" %}{{ default_email_body }}{% end_content_attribute %}", "emailNote": "", "emailTemplateMode": "DRAG_AND_DROP", "emailType": "AUTOMATED_EMAIL", "emailbodyPlaintext": "", "feedbackSurveyId": null, "flexAreas": {"main": {"boxed": false, "isSingleColumnFullWidth": false, "sections": [{"columns": [{"id": "column-0-1", "widgets": ["module-0-1-1"], "width": 12}], "id": "section-0", "style": {"backgroundColor": "#eaf0f6", "backgroundType": "CONTENT", "paddingBottom": "10px", "paddingTop": "10px"}}, {"columns": [{"id": "column-1-1", "widgets": ["module-1-1-1"], "width": 12}], "id": "section-1", "style": {"backgroundType": "CONTENT", "paddingBottom": "30px", "paddingTop": "30px"}}, {"columns": [{"id": "column-2-1", "widgets": ["module-2-1-1"], "width": 12}], "id": "section-2", "style": {"backgroundColor": "", "backgroundType": "CONTENT", "paddingBottom": "20px", "paddingTop": "20px"}}]}}, "freezeDate": 1634042970319, "fromName": "Team Airbyte", "hasContentAccessRules": false, "htmlTitle": "", "id": 42930862366, "isCreatedFomSandboxSync": false, "isGraymailSuppressionEnabled": false, "isInstanceLayoutPage": false, "isPublished": false, "isRecipientFatigueSuppressionEnabled": null, "language": "en", "lastEditSessionId": 1634042969643, "lastEditUpdateId": 0, "layoutSections": {}, "liveDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "mailingListsExcluded": [], "mailingListsIncluded": [], "maxRssEntries": 5, "metaDescription": "", "name": "Test subject (Test campaing - Clone)", "pageExpiryEnabled": false, "pageRedirected": false, "pastMabExperimentIds": [], "portalId": 8727216, "previewKey": "UmZGYZsU", "processingStatus": "UNDEFINED", "publishDate": 1634042970000, "publishImmediately": true, "publishedUrl": "", "replyTo": "integration-test@airbyte.io", "resolvedDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "rootMicId": null, "rssEmailByText": "By", "rssEmailClickThroughText": "Read more »", "rssEmailCommentText": "Comment »", "rssEmailEntryTemplateEnabled": false, "rssEmailImageMaxWidth": 0, "rssEmailUrl": "", "sections": {}, "securityState": "NONE", "slug": "-temporary-slug-fb53d6bf-1eb6-4ee6-90fe-610fc2569ea7", "smartEmailFields": {}, "state": "AUTOMATED_DRAFT", "styleSettings": {"background_color": "#EAF0F6", "background_image": null, "background_image_type": null, "body_border_color": "#EAF0F6", "body_border_color_choice": "BORDER_MANUAL", "body_border_width": "1", "body_color": "#ffffff", "color_picker_favorite1": null, "color_picker_favorite2": null, "color_picker_favorite3": null, "color_picker_favorite4": null, "color_picker_favorite5": null, "color_picker_favorite6": null, "email_body_padding": null, "email_body_width": null, "heading_one_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "28", "underline": null}, "heading_two_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "22", "underline": null}, "links_font": {"bold": false, "color": "#00a4bd", "font": null, "font_style": {}, "italic": false, "size": null, "underline": true}, "primary_accent_color": null, "primary_font": "Arial, sans-serif", "primary_font_color": "#23496d", "primary_font_line_height": null, "primary_font_size": "15", "secondary_accent_color": null, "secondary_font": "Arial, sans-serif", "secondary_font_color": "#23496d", "secondary_font_line_height": null, "secondary_font_size": "12", "use_email_client_default_settings": false, "user_module_defaults": {"button_email": {"background_color": "#00a4bd", "corner_radius": 8, "font": "Arial, sans-serif", "font_color": "#ffffff", "font_size": 16, "font_style": {"color": "#ffffff", "font": "Arial, sans-serif", "size": {"units": "px", "value": 16}, "styles": {"bold": false, "italic": false, "underline": false}}}, "email_divider": {"color": {"color": "#23496d", "opacity": 100}, "height": 1, "line_type": "solid"}}}, "subcategory": "automated", "subject": "Test subject", "subscription": 11890831, "subscriptionName": "Test subscription", "teamPerms": [], "templatePath": "@hubspot/email/dnd/welcome.html", "transactional": false, "translations": {}, "unpublishedAt": 0, "updated": 1634042970321, "updatedById": 12282590, "url": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-fb53d6bf-1eb6-4ee6-90fe-610fc2569ea7", "useRssHeadlineAsSubject": false, "userPerms": [], "vidsExcluded": [], "vidsIncluded": [], "visibleToAll": true}, "emitted_at": 1706192639131} {"stream": "owners", "data": {"id": "52550153", "email": "integration-test@airbyte.io", "firstName": "Team-1", "lastName": "Airbyte", "userId": 12282590, "createdAt": "2020-10-28T21:17:56.082Z", "updatedAt": "2023-01-31T00:25:34.448Z", "archived": false}, "emitted_at": 1697714250730} {"stream": "owners", "data": {"id": "65568071", "email": "test-integration-test-user1@airbyte.io", "firstName": "", "lastName": "", "userId": 23660227, "createdAt": "2021-03-15T11:00:50.053Z", "updatedAt": "2021-03-15T11:00:50.053Z", "archived": false}, "emitted_at": 1697714250731} {"stream": "owners", "data": {"id": "65568800", "email": "test-integration-test-user2@airbyte.io", "firstName": "", "lastName": "", "userId": 23660229, "createdAt": "2021-03-15T11:01:02.183Z", "updatedAt": "2021-03-15T11:01:02.183Z", "archived": false}, "emitted_at": 1697714250732} -{"stream": "products", "data": {"id": "646176421", "properties": {"amount": null, "createdate": "2021-02-23T20:03:18.336000+00:00", "description": null, "discount": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_avatar_filemanager_key": null, "hs_cost_of_goods_sold": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_folder_id": null, "hs_folder_name": null, "hs_images": null, "hs_lastmodifieddate": "2021-02-23T20:03:18.336000+00:00", "hs_merged_object_ids": null, "hs_object_id": 646176421, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_sku": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Test product", "price": 100, "quantity": null, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:03:18.336Z", "updatedAt": "2021-02-23T20:03:18.336Z", "archived": false, "properties_amount": null, "properties_createdate": "2021-02-23T20:03:18.336000+00:00", "properties_description": null, "properties_discount": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_cost_of_goods_sold": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_folder_id": null, "properties_hs_folder_name": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-02-23T20:03:18.336000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 646176421, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_sku": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Test product", "properties_price": 100, "properties_quantity": null, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1697714252635} -{"stream": "products", "data": {"id": "646176423", "properties": {"amount": null, "createdate": "2021-02-23T20:03:48.577000+00:00", "description": null, "discount": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_avatar_filemanager_key": null, "hs_cost_of_goods_sold": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_folder_id": 2430008, "hs_folder_name": "test folder", "hs_images": null, "hs_lastmodifieddate": "2021-02-23T20:03:48.577000+00:00", "hs_merged_object_ids": null, "hs_object_id": 646176423, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_sku": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Test product 1", "price": 123, "quantity": null, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:03:48.577Z", "updatedAt": "2021-02-23T20:03:48.577Z", "archived": false, "properties_amount": null, "properties_createdate": "2021-02-23T20:03:48.577000+00:00", "properties_description": null, "properties_discount": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_cost_of_goods_sold": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_folder_id": 2430008, "properties_hs_folder_name": "test folder", "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-02-23T20:03:48.577000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 646176423, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_sku": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Test product 1", "properties_price": 123, "properties_quantity": null, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1697714252637} -{"stream": "products", "data": {"id": "646316535", "properties": {"amount": null, "createdate": "2021-02-23T20:11:54.030000+00:00", "description": "baseball hat, large", "discount": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_avatar_filemanager_key": null, "hs_cost_of_goods_sold": 5, "hs_created_by_user_id": null, "hs_createdate": null, "hs_discount_percentage": null, "hs_folder_id": null, "hs_folder_name": null, "hs_images": null, "hs_lastmodifieddate": "2021-02-23T20:11:54.030000+00:00", "hs_merged_object_ids": null, "hs_object_id": 646316535, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_sku": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": true, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Green Hat", "price": 10, "quantity": null, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:11:54.030Z", "updatedAt": "2021-02-23T20:11:54.030Z", "archived": false, "properties_amount": null, "properties_createdate": "2021-02-23T20:11:54.030000+00:00", "properties_description": "baseball hat, large", "properties_discount": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_cost_of_goods_sold": 5, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_folder_id": null, "properties_hs_folder_name": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-02-23T20:11:54.030000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 646316535, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_sku": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": true, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Green Hat", "properties_price": 10, "properties_quantity": null, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1697714252638} +{"stream": "products", "data": {"id": "646176421", "properties": {"amount": null, "createdate": "2021-02-23T20:03:18.336000+00:00", "description": null, "discount": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_avatar_filemanager_key": null, "hs_cost_of_goods_sold": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_folder_id": null, "hs_folder_name": null, "hs_images": null, "hs_lastmodifieddate": "2021-02-23T20:03:18.336000+00:00", "hs_merged_object_ids": null, "hs_object_id": 646176421, "hs_object_source": "CRM_UI", "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_sku": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Test product", "price": 100, "quantity": null, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:03:18.336Z", "updatedAt": "2021-02-23T20:03:18.336Z", "archived": false, "properties_amount": null, "properties_createdate": "2021-02-23T20:03:18.336000+00:00", "properties_description": null, "properties_discount": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_cost_of_goods_sold": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_folder_id": null, "properties_hs_folder_name": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-02-23T20:03:18.336000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 646176421, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_sku": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Test product", "properties_price": 100, "properties_quantity": null, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1706192761055} +{"stream": "products", "data": {"id": "646176423", "properties": {"amount": null, "createdate": "2021-02-23T20:03:48.577000+00:00", "description": null, "discount": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_avatar_filemanager_key": null, "hs_cost_of_goods_sold": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_folder_id": 2430008, "hs_folder_name": "test folder", "hs_images": null, "hs_lastmodifieddate": "2021-02-23T20:03:48.577000+00:00", "hs_merged_object_ids": null, "hs_object_id": 646176423, "hs_object_source": "CRM_UI", "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_sku": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Test product 1", "price": 123, "quantity": null, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:03:48.577Z", "updatedAt": "2021-02-23T20:03:48.577Z", "archived": false, "properties_amount": null, "properties_createdate": "2021-02-23T20:03:48.577000+00:00", "properties_description": null, "properties_discount": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_cost_of_goods_sold": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_folder_id": 2430008, "properties_hs_folder_name": "test folder", "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-02-23T20:03:48.577000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 646176423, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_sku": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Test product 1", "properties_price": 123, "properties_quantity": null, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1706192761056} +{"stream": "products", "data": {"id": "646316535", "properties": {"amount": null, "createdate": "2021-02-23T20:11:54.030000+00:00", "description": "baseball hat, large", "discount": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_avatar_filemanager_key": null, "hs_cost_of_goods_sold": 5, "hs_created_by_user_id": null, "hs_createdate": null, "hs_discount_percentage": null, "hs_folder_id": null, "hs_folder_name": null, "hs_images": null, "hs_lastmodifieddate": "2021-02-23T20:11:54.030000+00:00", "hs_merged_object_ids": null, "hs_object_id": 646316535, "hs_object_source": "IMPORT", "hs_object_source_id": null, "hs_object_source_label": "IMPORT", "hs_object_source_user_id": null, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_sku": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": true, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Green Hat", "price": 10, "quantity": null, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:11:54.030Z", "updatedAt": "2021-02-23T20:11:54.030Z", "archived": false, "properties_amount": null, "properties_createdate": "2021-02-23T20:11:54.030000+00:00", "properties_description": "baseball hat, large", "properties_discount": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_cost_of_goods_sold": 5, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_folder_id": null, "properties_hs_folder_name": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-02-23T20:11:54.030000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 646316535, "properties_hs_object_source": "IMPORT", "properties_hs_object_source_id": null, "properties_hs_object_source_label": "IMPORT", "properties_hs_object_source_user_id": null, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_sku": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": true, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Green Hat", "properties_price": 10, "properties_quantity": null, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1706192761057} {"stream": "contacts_property_history", "data": {"value": "testo", "source-type": "CRM_UI", "source-id": "userId:12282590", "source-label": null, "updated-by-user-id": 12282590, "timestamp": 1700681340515, "selected": false, "property": "firstname", "vid": 2501, "portal-id": 8727216, "is-contact": true, "canonical-vid": 2501}, "emitted_at": 1701905506064} {"stream": "contacts_property_history", "data": {"value": "test", "source-type": "CRM_UI", "source-id": "userId:12282590", "source-label": null, "updated-by-user-id": 12282590, "timestamp": 1675120629904, "selected": false, "property": "firstname", "vid": 2501, "portal-id": 8727216, "is-contact": true, "canonical-vid": 2501}, "emitted_at": 1701905506064} {"stream": "companies_property_history", "data": {"name": "hs_analytics_latest_source_data_2", "value": "CRM_UI", "timestamp": 1657222285656, "sourceId": "RollupProperties", "source": "MIGRATION", "sourceVid": [], "property": "hs_analytics_latest_source_data_2", "companyId": 5000526215, "portalId": 8727216, "isDeleted": false}, "emitted_at": 1701905731242} @@ -58,9 +58,9 @@ {"stream": "subscription_changes", "data": {"timestamp": 1616173134301, "portalId": 8727216, "recipient": "0c90ecf5-629e-4fe4-8516-05f75636c3e3@gdpr-forgotten.hubspot.com", "normalizedEmailId": "0c90ecf5-629e-4fe4-8516-05f75636c3e3", "changes": [{"source": "SOURCE_HUBSPOT_CUSTOMER", "timestamp": 1616173134301, "portalId": 8727216, "causedByEvent": {"id": "d70b78b9-a411-4d3e-808b-fe931be35b43", "created": 1616173134301}, "changeType": "PORTAL_STATUS", "change": "SUBSCRIBED"}]}, "emitted_at": 1697714255435} {"stream": "subscription_changes", "data": {"timestamp": 1616173134301, "portalId": 8727216, "recipient": "0c90ecf5-629e-4fe4-8516-05f75636c3e3@gdpr-forgotten.hubspot.com", "normalizedEmailId": "0c90ecf5-629e-4fe4-8516-05f75636c3e3", "changes": [{"source": "SOURCE_HUBSPOT_CUSTOMER", "timestamp": 1616173134301, "subscriptionId": 10798197, "portalId": 8727216, "causedByEvent": {"id": "ff118718-786d-4a35-94f9-6bbd413654de", "created": 1616173134301}, "changeType": "SUBSCRIPTION_STATUS", "change": "SUBSCRIBED"}]}, "emitted_at": 1697714255436} {"stream": "subscription_changes", "data": {"timestamp": 1616173106737, "portalId": 8727216, "recipient": "0c90ecf5-629e-4fe4-8516-05f75636c3e3@gdpr-forgotten.hubspot.com", "normalizedEmailId": "0c90ecf5-629e-4fe4-8516-05f75636c3e3", "changes": [{"source": "SOURCE_HUBSPOT_CUSTOMER", "timestamp": 1616173106737, "portalId": 8727216, "causedByEvent": {"id": "24539f1f-0b20-4296-a5bf-6ba3bb9dc1b8", "created": 1616173106737}, "changeType": "PORTAL_STATUS", "change": "SUBSCRIBED"}]}, "emitted_at": 1697714255437} -{"stream": "tickets", "data": {"id": "312929579", "properties": {"closed_date": "2021-02-23T20:08:49.603000+00:00", "content": null, "created_by": null, "createdate": "2021-02-23T20:08:49.603000+00:00", "first_agent_reply_date": null, "hs_all_accessible_team_ids": null, "hs_all_associated_contact_companies": null, "hs_all_associated_contact_emails": null, "hs_all_associated_contact_firstnames": null, "hs_all_associated_contact_lastnames": null, "hs_all_associated_contact_mobilephones": null, "hs_all_associated_contact_phones": null, "hs_all_conversation_mentions": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignment_method": null, "hs_auto_generated_from_thread_id": null, "hs_conversations_originating_message_id": null, "hs_conversations_originating_thread_id": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_custom_inbox": null, "hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_2": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_3": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_4": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_2": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_3": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_4": null, "hs_external_object_ids": null, "hs_feedback_last_ces_follow_up": null, "hs_feedback_last_ces_rating": null, "hs_feedback_last_survey_date": null, "hs_file_upload": null, "hs_first_agent_message_sent_at": null, "hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "hs_in_helpdesk": null, "hs_inbox_id": null, "hs_is_visible_in_help_desk": null, "hs_last_email_activity": null, "hs_last_email_date": null, "hs_last_message_from_visitor": false, "hs_last_message_received_at": null, "hs_last_message_sent_at": null, "hs_lastactivitydate": null, "hs_lastcontacted": null, "hs_lastmodifieddate": "2021-02-23T20:08:53.371000+00:00", "hs_latest_message_seen_by_agent_ids": null, "hs_merged_object_ids": null, "hs_most_relevant_sla_status": null, "hs_most_relevant_sla_type": null, "hs_msteams_message_id": null, "hs_nextactivitydate": null, "hs_num_associated_companies": 0, "hs_num_associated_conversations": null, "hs_num_times_contacted": null, "hs_object_id": 312929579, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_originating_channel_instance_id": null, "hs_originating_email_engagement_id": null, "hs_originating_generic_channel_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "0", "hs_pipeline_stage": "4", "hs_primary_company": null, "hs_primary_company_id": null, "hs_primary_company_name": null, "hs_read_only": null, "hs_resolution": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_thread_ids_to_restore": null, "hs_ticket_category": null, "hs_ticket_id": 312929579, "hs_ticket_priority": "LOW", "hs_time_in_1": 0, "hs_time_in_2": 0, "hs_time_in_3": 0, "hs_time_in_4": 87748604133, "hs_time_to_close_sla_at": null, "hs_time_to_close_sla_status": null, "hs_time_to_first_response_sla_at": null, "hs_time_to_first_response_sla_status": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": true, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "last_engagement_date": null, "last_reply_date": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "nps_follow_up_answer": null, "nps_follow_up_question_version": null, "nps_score": null, "num_contacted_notes": null, "num_notes": null, "source_ref": null, "source_thread_id": null, "source_type": "CHAT", "subject": "Marketing Starter", "tags": null, "time_to_close": 0, "time_to_first_agent_reply": null}, "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, "properties_closed_date": "2021-02-23T20:08:49.603000+00:00", "properties_content": null, "properties_created_by": null, "properties_createdate": "2021-02-23T20:08:49.603000+00:00", "properties_first_agent_reply_date": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_associated_contact_companies": null, "properties_hs_all_associated_contact_emails": null, "properties_hs_all_associated_contact_firstnames": null, "properties_hs_all_associated_contact_lastnames": null, "properties_hs_all_associated_contact_mobilephones": null, "properties_hs_all_associated_contact_phones": null, "properties_hs_all_conversation_mentions": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignment_method": null, "properties_hs_auto_generated_from_thread_id": null, "properties_hs_conversations_originating_message_id": null, "properties_hs_conversations_originating_thread_id": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_custom_inbox": null, "properties_hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_2": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_3": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_4": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_2": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_3": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_4": null, "properties_hs_external_object_ids": null, "properties_hs_feedback_last_ces_follow_up": null, "properties_hs_feedback_last_ces_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_file_upload": null, "properties_hs_first_agent_message_sent_at": null, "properties_hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "properties_hs_in_helpdesk": null, "properties_hs_inbox_id": null, "properties_hs_is_visible_in_help_desk": null, "properties_hs_last_email_activity": null, "properties_hs_last_email_date": null, "properties_hs_last_message_from_visitor": false, "properties_hs_last_message_received_at": null, "properties_hs_last_message_sent_at": null, "properties_hs_lastactivitydate": null, "properties_hs_lastcontacted": null, "properties_hs_lastmodifieddate": "2021-02-23T20:08:53.371000+00:00", "properties_hs_latest_message_seen_by_agent_ids": null, "properties_hs_merged_object_ids": null, "properties_hs_most_relevant_sla_status": null, "properties_hs_most_relevant_sla_type": null, "properties_hs_msteams_message_id": null, "properties_hs_nextactivitydate": null, "properties_hs_num_associated_companies": 0, "properties_hs_num_associated_conversations": null, "properties_hs_num_times_contacted": null, "properties_hs_object_id": 312929579, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_originating_channel_instance_id": null, "properties_hs_originating_email_engagement_id": null, "properties_hs_originating_generic_channel_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "0", "properties_hs_pipeline_stage": "4", "properties_hs_primary_company": null, "properties_hs_primary_company_id": null, "properties_hs_primary_company_name": null, "properties_hs_read_only": null, "properties_hs_resolution": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_thread_ids_to_restore": null, "properties_hs_ticket_category": null, "properties_hs_ticket_id": 312929579, "properties_hs_ticket_priority": "LOW", "properties_hs_time_in_1": 0, "properties_hs_time_in_2": 0, "properties_hs_time_in_3": 0, "properties_hs_time_in_4": 87748604133, "properties_hs_time_to_close_sla_at": null, "properties_hs_time_to_close_sla_status": null, "properties_hs_time_to_first_response_sla_at": null, "properties_hs_time_to_first_response_sla_status": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": true, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_last_engagement_date": null, "properties_last_reply_date": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_nps_follow_up_answer": null, "properties_nps_follow_up_question_version": null, "properties_nps_score": null, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_source_ref": null, "properties_source_thread_id": null, "properties_source_type": "CHAT", "properties_subject": "Marketing Starter", "properties_tags": null, "properties_time_to_close": 0, "properties_time_to_first_agent_reply": null}, "emitted_at": 1701859534671} -{"stream": "tickets", "data": {"id": "312972611", "properties": {"closed_date": null, "content": null, "created_by": null, "createdate": "2021-02-23T20:08:49.603000+00:00", "first_agent_reply_date": null, "hs_all_accessible_team_ids": null, "hs_all_associated_contact_companies": null, "hs_all_associated_contact_emails": null, "hs_all_associated_contact_firstnames": null, "hs_all_associated_contact_lastnames": null, "hs_all_associated_contact_mobilephones": null, "hs_all_associated_contact_phones": null, "hs_all_conversation_mentions": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignment_method": null, "hs_auto_generated_from_thread_id": null, "hs_conversations_originating_message_id": null, "hs_conversations_originating_thread_id": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_custom_inbox": null, "hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_2": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_3": null, "hs_date_entered_4": null, "hs_date_exited_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_2": null, "hs_date_exited_3": null, "hs_date_exited_4": null, "hs_external_object_ids": null, "hs_feedback_last_ces_follow_up": null, "hs_feedback_last_ces_rating": null, "hs_feedback_last_survey_date": null, "hs_file_upload": null, "hs_first_agent_message_sent_at": null, "hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "hs_in_helpdesk": null, "hs_inbox_id": null, "hs_is_visible_in_help_desk": null, "hs_last_email_activity": null, "hs_last_email_date": null, "hs_last_message_from_visitor": false, "hs_last_message_received_at": null, "hs_last_message_sent_at": null, "hs_lastactivitydate": null, "hs_lastcontacted": null, "hs_lastmodifieddate": "2021-02-23T20:08:52.663000+00:00", "hs_latest_message_seen_by_agent_ids": null, "hs_merged_object_ids": null, "hs_most_relevant_sla_status": null, "hs_most_relevant_sla_type": null, "hs_msteams_message_id": null, "hs_nextactivitydate": null, "hs_num_associated_companies": 0, "hs_num_associated_conversations": null, "hs_num_times_contacted": null, "hs_object_id": 312972611, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_originating_channel_instance_id": null, "hs_originating_email_engagement_id": null, "hs_originating_generic_channel_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "0", "hs_pipeline_stage": "2", "hs_primary_company": null, "hs_primary_company_id": null, "hs_primary_company_name": null, "hs_read_only": null, "hs_resolution": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_thread_ids_to_restore": null, "hs_ticket_category": null, "hs_ticket_id": 312972611, "hs_ticket_priority": "LOW", "hs_time_in_1": 0, "hs_time_in_2": 87748604132, "hs_time_in_3": null, "hs_time_in_4": null, "hs_time_to_close_sla_at": null, "hs_time_to_close_sla_status": null, "hs_time_to_first_response_sla_at": null, "hs_time_to_first_response_sla_status": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": true, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "last_engagement_date": null, "last_reply_date": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "nps_follow_up_answer": null, "nps_follow_up_question_version": null, "nps_score": null, "num_contacted_notes": null, "num_notes": null, "source_ref": null, "source_thread_id": null, "source_type": "FORM", "subject": "Sales Starter", "tags": null, "time_to_close": null, "time_to_first_agent_reply": null}, "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:52.663Z", "archived": false, "properties_closed_date": null, "properties_content": null, "properties_created_by": null, "properties_createdate": "2021-02-23T20:08:49.603000+00:00", "properties_first_agent_reply_date": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_associated_contact_companies": null, "properties_hs_all_associated_contact_emails": null, "properties_hs_all_associated_contact_firstnames": null, "properties_hs_all_associated_contact_lastnames": null, "properties_hs_all_associated_contact_mobilephones": null, "properties_hs_all_associated_contact_phones": null, "properties_hs_all_conversation_mentions": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignment_method": null, "properties_hs_auto_generated_from_thread_id": null, "properties_hs_conversations_originating_message_id": null, "properties_hs_conversations_originating_thread_id": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_custom_inbox": null, "properties_hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_2": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_3": null, "properties_hs_date_entered_4": null, "properties_hs_date_exited_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_2": null, "properties_hs_date_exited_3": null, "properties_hs_date_exited_4": null, "properties_hs_external_object_ids": null, "properties_hs_feedback_last_ces_follow_up": null, "properties_hs_feedback_last_ces_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_file_upload": null, "properties_hs_first_agent_message_sent_at": null, "properties_hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "properties_hs_in_helpdesk": null, "properties_hs_inbox_id": null, "properties_hs_is_visible_in_help_desk": null, "properties_hs_last_email_activity": null, "properties_hs_last_email_date": null, "properties_hs_last_message_from_visitor": false, "properties_hs_last_message_received_at": null, "properties_hs_last_message_sent_at": null, "properties_hs_lastactivitydate": null, "properties_hs_lastcontacted": null, "properties_hs_lastmodifieddate": "2021-02-23T20:08:52.663000+00:00", "properties_hs_latest_message_seen_by_agent_ids": null, "properties_hs_merged_object_ids": null, "properties_hs_most_relevant_sla_status": null, "properties_hs_most_relevant_sla_type": null, "properties_hs_msteams_message_id": null, "properties_hs_nextactivitydate": null, "properties_hs_num_associated_companies": 0, "properties_hs_num_associated_conversations": null, "properties_hs_num_times_contacted": null, "properties_hs_object_id": 312972611, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_originating_channel_instance_id": null, "properties_hs_originating_email_engagement_id": null, "properties_hs_originating_generic_channel_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "0", "properties_hs_pipeline_stage": "2", "properties_hs_primary_company": null, "properties_hs_primary_company_id": null, "properties_hs_primary_company_name": null, "properties_hs_read_only": null, "properties_hs_resolution": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_thread_ids_to_restore": null, "properties_hs_ticket_category": null, "properties_hs_ticket_id": 312972611, "properties_hs_ticket_priority": "LOW", "properties_hs_time_in_1": 0, "properties_hs_time_in_2": 87748604132, "properties_hs_time_in_3": null, "properties_hs_time_in_4": null, "properties_hs_time_to_close_sla_at": null, "properties_hs_time_to_close_sla_status": null, "properties_hs_time_to_first_response_sla_at": null, "properties_hs_time_to_first_response_sla_status": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": true, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_last_engagement_date": null, "properties_last_reply_date": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_nps_follow_up_answer": null, "properties_nps_follow_up_question_version": null, "properties_nps_score": null, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_source_ref": null, "properties_source_thread_id": null, "properties_source_type": "FORM", "properties_subject": "Sales Starter", "properties_tags": null, "properties_time_to_close": null, "properties_time_to_first_agent_reply": null}, "emitted_at": 1701859534672} -{"stream": "tickets", "data": {"id": "312975112", "properties": {"closed_date": null, "content": null, "created_by": null, "createdate": "2021-02-23T20:08:49.603000+00:00", "first_agent_reply_date": null, "hs_all_accessible_team_ids": null, "hs_all_associated_contact_companies": null, "hs_all_associated_contact_emails": null, "hs_all_associated_contact_firstnames": null, "hs_all_associated_contact_lastnames": null, "hs_all_associated_contact_mobilephones": null, "hs_all_associated_contact_phones": null, "hs_all_conversation_mentions": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignment_method": null, "hs_auto_generated_from_thread_id": null, "hs_conversations_originating_message_id": null, "hs_conversations_originating_thread_id": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_custom_inbox": null, "hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_2": null, "hs_date_entered_3": null, "hs_date_entered_4": null, "hs_date_exited_1": null, "hs_date_exited_2": null, "hs_date_exited_3": null, "hs_date_exited_4": null, "hs_external_object_ids": null, "hs_feedback_last_ces_follow_up": null, "hs_feedback_last_ces_rating": null, "hs_feedback_last_survey_date": null, "hs_file_upload": null, "hs_first_agent_message_sent_at": null, "hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "hs_in_helpdesk": null, "hs_inbox_id": null, "hs_is_visible_in_help_desk": null, "hs_last_email_activity": null, "hs_last_email_date": null, "hs_last_message_from_visitor": false, "hs_last_message_received_at": null, "hs_last_message_sent_at": null, "hs_lastactivitydate": null, "hs_lastcontacted": null, "hs_lastmodifieddate": "2021-02-23T20:08:52.515000+00:00", "hs_latest_message_seen_by_agent_ids": null, "hs_merged_object_ids": null, "hs_most_relevant_sla_status": null, "hs_most_relevant_sla_type": null, "hs_msteams_message_id": null, "hs_nextactivitydate": null, "hs_num_associated_companies": 0, "hs_num_associated_conversations": null, "hs_num_times_contacted": null, "hs_object_id": 312975112, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_originating_channel_instance_id": null, "hs_originating_email_engagement_id": null, "hs_originating_generic_channel_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "0", "hs_pipeline_stage": "1", "hs_primary_company": null, "hs_primary_company_id": null, "hs_primary_company_name": null, "hs_read_only": null, "hs_resolution": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_thread_ids_to_restore": null, "hs_ticket_category": null, "hs_ticket_id": 312975112, "hs_ticket_priority": "MEDIUM", "hs_time_in_1": 87748604134, "hs_time_in_2": null, "hs_time_in_3": null, "hs_time_in_4": null, "hs_time_to_close_sla_at": null, "hs_time_to_close_sla_status": null, "hs_time_to_first_response_sla_at": null, "hs_time_to_first_response_sla_status": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": true, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "last_engagement_date": null, "last_reply_date": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "nps_follow_up_answer": null, "nps_follow_up_question_version": null, "nps_score": null, "num_contacted_notes": null, "num_notes": null, "source_ref": null, "source_thread_id": null, "source_type": "PHONE", "subject": "Free CRM", "tags": null, "time_to_close": null, "time_to_first_agent_reply": null}, "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:52.515Z", "archived": false, "properties_closed_date": null, "properties_content": null, "properties_created_by": null, "properties_createdate": "2021-02-23T20:08:49.603000+00:00", "properties_first_agent_reply_date": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_associated_contact_companies": null, "properties_hs_all_associated_contact_emails": null, "properties_hs_all_associated_contact_firstnames": null, "properties_hs_all_associated_contact_lastnames": null, "properties_hs_all_associated_contact_mobilephones": null, "properties_hs_all_associated_contact_phones": null, "properties_hs_all_conversation_mentions": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignment_method": null, "properties_hs_auto_generated_from_thread_id": null, "properties_hs_conversations_originating_message_id": null, "properties_hs_conversations_originating_thread_id": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_custom_inbox": null, "properties_hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_2": null, "properties_hs_date_entered_3": null, "properties_hs_date_entered_4": null, "properties_hs_date_exited_1": null, "properties_hs_date_exited_2": null, "properties_hs_date_exited_3": null, "properties_hs_date_exited_4": null, "properties_hs_external_object_ids": null, "properties_hs_feedback_last_ces_follow_up": null, "properties_hs_feedback_last_ces_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_file_upload": null, "properties_hs_first_agent_message_sent_at": null, "properties_hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "properties_hs_in_helpdesk": null, "properties_hs_inbox_id": null, "properties_hs_is_visible_in_help_desk": null, "properties_hs_last_email_activity": null, "properties_hs_last_email_date": null, "properties_hs_last_message_from_visitor": false, "properties_hs_last_message_received_at": null, "properties_hs_last_message_sent_at": null, "properties_hs_lastactivitydate": null, "properties_hs_lastcontacted": null, "properties_hs_lastmodifieddate": "2021-02-23T20:08:52.515000+00:00", "properties_hs_latest_message_seen_by_agent_ids": null, "properties_hs_merged_object_ids": null, "properties_hs_most_relevant_sla_status": null, "properties_hs_most_relevant_sla_type": null, "properties_hs_msteams_message_id": null, "properties_hs_nextactivitydate": null, "properties_hs_num_associated_companies": 0, "properties_hs_num_associated_conversations": null, "properties_hs_num_times_contacted": null, "properties_hs_object_id": 312975112, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_originating_channel_instance_id": null, "properties_hs_originating_email_engagement_id": null, "properties_hs_originating_generic_channel_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "0", "properties_hs_pipeline_stage": "1", "properties_hs_primary_company": null, "properties_hs_primary_company_id": null, "properties_hs_primary_company_name": null, "properties_hs_read_only": null, "properties_hs_resolution": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_thread_ids_to_restore": null, "properties_hs_ticket_category": null, "properties_hs_ticket_id": 312975112, "properties_hs_ticket_priority": "MEDIUM", "properties_hs_time_in_1": 87748604134, "properties_hs_time_in_2": null, "properties_hs_time_in_3": null, "properties_hs_time_in_4": null, "properties_hs_time_to_close_sla_at": null, "properties_hs_time_to_close_sla_status": null, "properties_hs_time_to_first_response_sla_at": null, "properties_hs_time_to_first_response_sla_status": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": true, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_last_engagement_date": null, "properties_last_reply_date": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_nps_follow_up_answer": null, "properties_nps_follow_up_question_version": null, "properties_nps_score": null, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_source_ref": null, "properties_source_thread_id": null, "properties_source_type": "PHONE", "properties_subject": "Free CRM", "properties_tags": null, "properties_time_to_close": null, "properties_time_to_first_agent_reply": null}, "emitted_at": 1701859534673} +{"stream": "tickets", "data": {"id": "312929579", "properties": {"closed_date": "2021-02-23T20:08:49.603000+00:00", "content": null, "created_by": null, "createdate": "2021-02-23T20:08:49.603000+00:00", "first_agent_reply_date": null, "hs_all_accessible_team_ids": null, "hs_all_associated_contact_companies": null, "hs_all_associated_contact_emails": null, "hs_all_associated_contact_firstnames": null, "hs_all_associated_contact_lastnames": null, "hs_all_associated_contact_mobilephones": null, "hs_all_associated_contact_phones": null, "hs_all_conversation_mentions": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignment_method": null, "hs_auto_generated_from_thread_id": null, "hs_conversations_originating_message_id": null, "hs_conversations_originating_thread_id": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_custom_inbox": null, "hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_2": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_3": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_4": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_2": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_3": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_4": null, "hs_external_object_ids": null, "hs_feedback_last_ces_follow_up": null, "hs_feedback_last_ces_rating": null, "hs_feedback_last_survey_date": null, "hs_file_upload": null, "hs_first_agent_message_sent_at": null, "hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "hs_in_helpdesk": null, "hs_inbox_id": null, "hs_is_visible_in_help_desk": null, "hs_last_email_activity": null, "hs_last_email_date": null, "hs_last_message_from_visitor": false, "hs_last_message_received_at": null, "hs_last_message_sent_at": null, "hs_lastactivitydate": null, "hs_lastcontacted": null, "hs_lastmodifieddate": "2021-02-23T20:08:53.371000+00:00", "hs_latest_message_seen_by_agent_ids": null, "hs_merged_object_ids": null, "hs_most_relevant_sla_status": null, "hs_most_relevant_sla_type": null, "hs_msteams_message_id": null, "hs_nextactivitydate": null, "hs_num_associated_companies": 0, "hs_num_associated_conversations": null, "hs_num_times_contacted": null, "hs_object_id": 312929579, "hs_object_source": "IMPORT", "hs_object_source_id": null, "hs_object_source_label": "IMPORT", "hs_object_source_user_id": null, "hs_originating_email_engagement_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "0", "hs_pipeline_stage": "4", "hs_primary_company": null, "hs_primary_company_id": null, "hs_primary_company_name": null, "hs_read_only": null, "hs_resolution": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_thread_ids_to_restore": null, "hs_ticket_category": null, "hs_ticket_id": 312929579, "hs_ticket_priority": "LOW", "hs_time_in_1": 0, "hs_time_in_2": 0, "hs_time_in_3": 0, "hs_time_in_4": 92081969615, "hs_time_to_close_sla_at": null, "hs_time_to_close_sla_status": null, "hs_time_to_first_response_sla_at": null, "hs_time_to_first_response_sla_status": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": true, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "last_engagement_date": null, "last_reply_date": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "nps_follow_up_answer": null, "nps_follow_up_question_version": null, "nps_score": null, "num_contacted_notes": null, "num_notes": null, "source_ref": null, "source_thread_id": null, "source_type": "CHAT", "subject": "Marketing Starter", "tags": null, "time_to_close": 0, "time_to_first_agent_reply": null}, "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, "properties_closed_date": "2021-02-23T20:08:49.603000+00:00", "properties_content": null, "properties_created_by": null, "properties_createdate": "2021-02-23T20:08:49.603000+00:00", "properties_first_agent_reply_date": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_associated_contact_companies": null, "properties_hs_all_associated_contact_emails": null, "properties_hs_all_associated_contact_firstnames": null, "properties_hs_all_associated_contact_lastnames": null, "properties_hs_all_associated_contact_mobilephones": null, "properties_hs_all_associated_contact_phones": null, "properties_hs_all_conversation_mentions": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignment_method": null, "properties_hs_auto_generated_from_thread_id": null, "properties_hs_conversations_originating_message_id": null, "properties_hs_conversations_originating_thread_id": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_custom_inbox": null, "properties_hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_2": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_3": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_4": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_2": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_3": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_4": null, "properties_hs_external_object_ids": null, "properties_hs_feedback_last_ces_follow_up": null, "properties_hs_feedback_last_ces_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_file_upload": null, "properties_hs_first_agent_message_sent_at": null, "properties_hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "properties_hs_in_helpdesk": null, "properties_hs_inbox_id": null, "properties_hs_is_visible_in_help_desk": null, "properties_hs_last_email_activity": null, "properties_hs_last_email_date": null, "properties_hs_last_message_from_visitor": false, "properties_hs_last_message_received_at": null, "properties_hs_last_message_sent_at": null, "properties_hs_lastactivitydate": null, "properties_hs_lastcontacted": null, "properties_hs_lastmodifieddate": "2021-02-23T20:08:53.371000+00:00", "properties_hs_latest_message_seen_by_agent_ids": null, "properties_hs_merged_object_ids": null, "properties_hs_most_relevant_sla_status": null, "properties_hs_most_relevant_sla_type": null, "properties_hs_msteams_message_id": null, "properties_hs_nextactivitydate": null, "properties_hs_num_associated_companies": 0, "properties_hs_num_associated_conversations": null, "properties_hs_num_times_contacted": null, "properties_hs_object_id": 312929579, "properties_hs_object_source": "IMPORT", "properties_hs_object_source_id": null, "properties_hs_object_source_label": "IMPORT", "properties_hs_object_source_user_id": null, "properties_hs_originating_email_engagement_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "0", "properties_hs_pipeline_stage": "4", "properties_hs_primary_company": null, "properties_hs_primary_company_id": null, "properties_hs_primary_company_name": null, "properties_hs_read_only": null, "properties_hs_resolution": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_thread_ids_to_restore": null, "properties_hs_ticket_category": null, "properties_hs_ticket_id": 312929579, "properties_hs_ticket_priority": "LOW", "properties_hs_time_in_1": 0, "properties_hs_time_in_2": 0, "properties_hs_time_in_3": 0, "properties_hs_time_in_4": 92081969615, "properties_hs_time_to_close_sla_at": null, "properties_hs_time_to_close_sla_status": null, "properties_hs_time_to_first_response_sla_at": null, "properties_hs_time_to_first_response_sla_status": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": true, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_last_engagement_date": null, "properties_last_reply_date": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_nps_follow_up_answer": null, "properties_nps_follow_up_question_version": null, "properties_nps_score": null, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_source_ref": null, "properties_source_thread_id": null, "properties_source_type": "CHAT", "properties_subject": "Marketing Starter", "properties_tags": null, "properties_time_to_close": 0, "properties_time_to_first_agent_reply": null}, "emitted_at": 1706192899343} +{"stream": "tickets", "data": {"id": "312972611", "properties": {"closed_date": null, "content": null, "created_by": null, "createdate": "2021-02-23T20:08:49.603000+00:00", "first_agent_reply_date": null, "hs_all_accessible_team_ids": null, "hs_all_associated_contact_companies": null, "hs_all_associated_contact_emails": null, "hs_all_associated_contact_firstnames": null, "hs_all_associated_contact_lastnames": null, "hs_all_associated_contact_mobilephones": null, "hs_all_associated_contact_phones": null, "hs_all_conversation_mentions": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignment_method": null, "hs_auto_generated_from_thread_id": null, "hs_conversations_originating_message_id": null, "hs_conversations_originating_thread_id": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_custom_inbox": null, "hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_2": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_3": null, "hs_date_entered_4": null, "hs_date_exited_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_2": null, "hs_date_exited_3": null, "hs_date_exited_4": null, "hs_external_object_ids": null, "hs_feedback_last_ces_follow_up": null, "hs_feedback_last_ces_rating": null, "hs_feedback_last_survey_date": null, "hs_file_upload": null, "hs_first_agent_message_sent_at": null, "hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "hs_in_helpdesk": null, "hs_inbox_id": null, "hs_is_visible_in_help_desk": null, "hs_last_email_activity": null, "hs_last_email_date": null, "hs_last_message_from_visitor": false, "hs_last_message_received_at": null, "hs_last_message_sent_at": null, "hs_lastactivitydate": null, "hs_lastcontacted": null, "hs_lastmodifieddate": "2021-02-23T20:08:52.663000+00:00", "hs_latest_message_seen_by_agent_ids": null, "hs_merged_object_ids": null, "hs_most_relevant_sla_status": null, "hs_most_relevant_sla_type": null, "hs_msteams_message_id": null, "hs_nextactivitydate": null, "hs_num_associated_companies": 0, "hs_num_associated_conversations": null, "hs_num_times_contacted": null, "hs_object_id": 312972611, "hs_object_source": "IMPORT", "hs_object_source_id": null, "hs_object_source_label": "IMPORT", "hs_object_source_user_id": null, "hs_originating_email_engagement_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "0", "hs_pipeline_stage": "2", "hs_primary_company": null, "hs_primary_company_id": null, "hs_primary_company_name": null, "hs_read_only": null, "hs_resolution": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_thread_ids_to_restore": null, "hs_ticket_category": null, "hs_ticket_id": 312972611, "hs_ticket_priority": "LOW", "hs_time_in_1": 0, "hs_time_in_2": 92081969613, "hs_time_in_3": null, "hs_time_in_4": null, "hs_time_to_close_sla_at": null, "hs_time_to_close_sla_status": null, "hs_time_to_first_response_sla_at": null, "hs_time_to_first_response_sla_status": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": true, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "last_engagement_date": null, "last_reply_date": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "nps_follow_up_answer": null, "nps_follow_up_question_version": null, "nps_score": null, "num_contacted_notes": null, "num_notes": null, "source_ref": null, "source_thread_id": null, "source_type": "FORM", "subject": "Sales Starter", "tags": null, "time_to_close": null, "time_to_first_agent_reply": null}, "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:52.663Z", "archived": false, "properties_closed_date": null, "properties_content": null, "properties_created_by": null, "properties_createdate": "2021-02-23T20:08:49.603000+00:00", "properties_first_agent_reply_date": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_associated_contact_companies": null, "properties_hs_all_associated_contact_emails": null, "properties_hs_all_associated_contact_firstnames": null, "properties_hs_all_associated_contact_lastnames": null, "properties_hs_all_associated_contact_mobilephones": null, "properties_hs_all_associated_contact_phones": null, "properties_hs_all_conversation_mentions": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignment_method": null, "properties_hs_auto_generated_from_thread_id": null, "properties_hs_conversations_originating_message_id": null, "properties_hs_conversations_originating_thread_id": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_custom_inbox": null, "properties_hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_2": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_3": null, "properties_hs_date_entered_4": null, "properties_hs_date_exited_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_2": null, "properties_hs_date_exited_3": null, "properties_hs_date_exited_4": null, "properties_hs_external_object_ids": null, "properties_hs_feedback_last_ces_follow_up": null, "properties_hs_feedback_last_ces_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_file_upload": null, "properties_hs_first_agent_message_sent_at": null, "properties_hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "properties_hs_in_helpdesk": null, "properties_hs_inbox_id": null, "properties_hs_is_visible_in_help_desk": null, "properties_hs_last_email_activity": null, "properties_hs_last_email_date": null, "properties_hs_last_message_from_visitor": false, "properties_hs_last_message_received_at": null, "properties_hs_last_message_sent_at": null, "properties_hs_lastactivitydate": null, "properties_hs_lastcontacted": null, "properties_hs_lastmodifieddate": "2021-02-23T20:08:52.663000+00:00", "properties_hs_latest_message_seen_by_agent_ids": null, "properties_hs_merged_object_ids": null, "properties_hs_most_relevant_sla_status": null, "properties_hs_most_relevant_sla_type": null, "properties_hs_msteams_message_id": null, "properties_hs_nextactivitydate": null, "properties_hs_num_associated_companies": 0, "properties_hs_num_associated_conversations": null, "properties_hs_num_times_contacted": null, "properties_hs_object_id": 312972611, "properties_hs_object_source": "IMPORT", "properties_hs_object_source_id": null, "properties_hs_object_source_label": "IMPORT", "properties_hs_object_source_user_id": null, "properties_hs_originating_email_engagement_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "0", "properties_hs_pipeline_stage": "2", "properties_hs_primary_company": null, "properties_hs_primary_company_id": null, "properties_hs_primary_company_name": null, "properties_hs_read_only": null, "properties_hs_resolution": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_thread_ids_to_restore": null, "properties_hs_ticket_category": null, "properties_hs_ticket_id": 312972611, "properties_hs_ticket_priority": "LOW", "properties_hs_time_in_1": 0, "properties_hs_time_in_2": 92081969613, "properties_hs_time_in_3": null, "properties_hs_time_in_4": null, "properties_hs_time_to_close_sla_at": null, "properties_hs_time_to_close_sla_status": null, "properties_hs_time_to_first_response_sla_at": null, "properties_hs_time_to_first_response_sla_status": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": true, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_last_engagement_date": null, "properties_last_reply_date": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_nps_follow_up_answer": null, "properties_nps_follow_up_question_version": null, "properties_nps_score": null, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_source_ref": null, "properties_source_thread_id": null, "properties_source_type": "FORM", "properties_subject": "Sales Starter", "properties_tags": null, "properties_time_to_close": null, "properties_time_to_first_agent_reply": null}, "emitted_at": 1706192899345} +{"stream": "tickets", "data": {"id": "312975112", "properties": {"closed_date": null, "content": null, "created_by": null, "createdate": "2021-02-23T20:08:49.603000+00:00", "first_agent_reply_date": null, "hs_all_accessible_team_ids": null, "hs_all_associated_contact_companies": null, "hs_all_associated_contact_emails": null, "hs_all_associated_contact_firstnames": null, "hs_all_associated_contact_lastnames": null, "hs_all_associated_contact_mobilephones": null, "hs_all_associated_contact_phones": null, "hs_all_conversation_mentions": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignment_method": null, "hs_auto_generated_from_thread_id": null, "hs_conversations_originating_message_id": null, "hs_conversations_originating_thread_id": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_custom_inbox": null, "hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_2": null, "hs_date_entered_3": null, "hs_date_entered_4": null, "hs_date_exited_1": null, "hs_date_exited_2": null, "hs_date_exited_3": null, "hs_date_exited_4": null, "hs_external_object_ids": null, "hs_feedback_last_ces_follow_up": null, "hs_feedback_last_ces_rating": null, "hs_feedback_last_survey_date": null, "hs_file_upload": null, "hs_first_agent_message_sent_at": null, "hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "hs_in_helpdesk": null, "hs_inbox_id": null, "hs_is_visible_in_help_desk": null, "hs_last_email_activity": null, "hs_last_email_date": null, "hs_last_message_from_visitor": false, "hs_last_message_received_at": null, "hs_last_message_sent_at": null, "hs_lastactivitydate": null, "hs_lastcontacted": null, "hs_lastmodifieddate": "2021-02-23T20:08:52.515000+00:00", "hs_latest_message_seen_by_agent_ids": null, "hs_merged_object_ids": null, "hs_most_relevant_sla_status": null, "hs_most_relevant_sla_type": null, "hs_msteams_message_id": null, "hs_nextactivitydate": null, "hs_num_associated_companies": 0, "hs_num_associated_conversations": null, "hs_num_times_contacted": null, "hs_object_id": 312975112, "hs_object_source": "IMPORT", "hs_object_source_id": null, "hs_object_source_label": "IMPORT", "hs_object_source_user_id": null, "hs_originating_email_engagement_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "0", "hs_pipeline_stage": "1", "hs_primary_company": null, "hs_primary_company_id": null, "hs_primary_company_name": null, "hs_read_only": null, "hs_resolution": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_thread_ids_to_restore": null, "hs_ticket_category": null, "hs_ticket_id": 312975112, "hs_ticket_priority": "MEDIUM", "hs_time_in_1": 92081969614, "hs_time_in_2": null, "hs_time_in_3": null, "hs_time_in_4": null, "hs_time_to_close_sla_at": null, "hs_time_to_close_sla_status": null, "hs_time_to_first_response_sla_at": null, "hs_time_to_first_response_sla_status": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": true, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "last_engagement_date": null, "last_reply_date": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "nps_follow_up_answer": null, "nps_follow_up_question_version": null, "nps_score": null, "num_contacted_notes": null, "num_notes": null, "source_ref": null, "source_thread_id": null, "source_type": "PHONE", "subject": "Free CRM", "tags": null, "time_to_close": null, "time_to_first_agent_reply": null}, "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:52.515Z", "archived": false, "properties_closed_date": null, "properties_content": null, "properties_created_by": null, "properties_createdate": "2021-02-23T20:08:49.603000+00:00", "properties_first_agent_reply_date": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_associated_contact_companies": null, "properties_hs_all_associated_contact_emails": null, "properties_hs_all_associated_contact_firstnames": null, "properties_hs_all_associated_contact_lastnames": null, "properties_hs_all_associated_contact_mobilephones": null, "properties_hs_all_associated_contact_phones": null, "properties_hs_all_conversation_mentions": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignment_method": null, "properties_hs_auto_generated_from_thread_id": null, "properties_hs_conversations_originating_message_id": null, "properties_hs_conversations_originating_thread_id": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_custom_inbox": null, "properties_hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_2": null, "properties_hs_date_entered_3": null, "properties_hs_date_entered_4": null, "properties_hs_date_exited_1": null, "properties_hs_date_exited_2": null, "properties_hs_date_exited_3": null, "properties_hs_date_exited_4": null, "properties_hs_external_object_ids": null, "properties_hs_feedback_last_ces_follow_up": null, "properties_hs_feedback_last_ces_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_file_upload": null, "properties_hs_first_agent_message_sent_at": null, "properties_hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "properties_hs_in_helpdesk": null, "properties_hs_inbox_id": null, "properties_hs_is_visible_in_help_desk": null, "properties_hs_last_email_activity": null, "properties_hs_last_email_date": null, "properties_hs_last_message_from_visitor": false, "properties_hs_last_message_received_at": null, "properties_hs_last_message_sent_at": null, "properties_hs_lastactivitydate": null, "properties_hs_lastcontacted": null, "properties_hs_lastmodifieddate": "2021-02-23T20:08:52.515000+00:00", "properties_hs_latest_message_seen_by_agent_ids": null, "properties_hs_merged_object_ids": null, "properties_hs_most_relevant_sla_status": null, "properties_hs_most_relevant_sla_type": null, "properties_hs_msteams_message_id": null, "properties_hs_nextactivitydate": null, "properties_hs_num_associated_companies": 0, "properties_hs_num_associated_conversations": null, "properties_hs_num_times_contacted": null, "properties_hs_object_id": 312975112, "properties_hs_object_source": "IMPORT", "properties_hs_object_source_id": null, "properties_hs_object_source_label": "IMPORT", "properties_hs_object_source_user_id": null, "properties_hs_originating_email_engagement_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "0", "properties_hs_pipeline_stage": "1", "properties_hs_primary_company": null, "properties_hs_primary_company_id": null, "properties_hs_primary_company_name": null, "properties_hs_read_only": null, "properties_hs_resolution": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_thread_ids_to_restore": null, "properties_hs_ticket_category": null, "properties_hs_ticket_id": 312975112, "properties_hs_ticket_priority": "MEDIUM", "properties_hs_time_in_1": 92081969614, "properties_hs_time_in_2": null, "properties_hs_time_in_3": null, "properties_hs_time_in_4": null, "properties_hs_time_to_close_sla_at": null, "properties_hs_time_to_close_sla_status": null, "properties_hs_time_to_first_response_sla_at": null, "properties_hs_time_to_first_response_sla_status": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": true, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_last_engagement_date": null, "properties_last_reply_date": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_nps_follow_up_answer": null, "properties_nps_follow_up_question_version": null, "properties_nps_score": null, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_source_ref": null, "properties_source_thread_id": null, "properties_source_type": "PHONE", "properties_subject": "Free CRM", "properties_tags": null, "properties_time_to_close": null, "properties_time_to_first_agent_reply": null}, "emitted_at": 1706192899346} {"stream": "workflows", "data": {"migrationStatus": {"portalId": 8727216, "workflowId": 21058115, "migrationStatus": "EXECUTION_MIGRATED", "enrollmentMigrationStatus": "PLATFORM_OWNED", "platformOwnsActions": true, "lastSuccessfulMigrationTimestamp": null, "enrollmentMigrationTimestamp": null, "flowId": 50206671}, "name": "Test Workflow", "id": 21058115, "type": "DRIP_DELAY", "enabled": false, "creationSource": {"sourceApplication": {"source": "DIRECT_API"}, "createdAt": 1610635826795}, "updateSource": {"sourceApplication": {"source": "DIRECT_API", "serviceName": "AutomationPlatformService-web_BackfillILSListIds"}, "updatedAt": 1611847907577}, "contactListIds": {"enrolled": 12, "active": 13, "completed": 14, "succeeded": 15}, "personaTagIds": [], "contactCounts": {"active": 0, "enrolled": 0}, "portalId": 8727216, "insertedAt": 1610635826921, "updatedAt": 1611847907577, "contactListIds_enrolled": 12, "contactListIds_active": 13, "contactListIds_completed": 14, "contactListIds_succeeded": 15}, "emitted_at": 1697714264418} {"stream": "workflows", "data": {"migrationStatus": {"portalId": 8727216, "workflowId": 21058121, "migrationStatus": "EXECUTION_MIGRATED", "enrollmentMigrationStatus": "PLATFORM_OWNED", "platformOwnsActions": true, "lastSuccessfulMigrationTimestamp": null, "enrollmentMigrationTimestamp": null, "flowId": 50205684}, "name": "Test Workflow 1", "id": 21058121, "type": "DRIP_DELAY", "enabled": false, "creationSource": {"sourceApplication": {"source": "DIRECT_API"}, "createdAt": 1610635850713}, "updateSource": {"sourceApplication": {"source": "DIRECT_API", "serviceName": "AutomationPlatformService-web_BackfillILSListIds"}, "updatedAt": 1611847907579}, "contactListIds": {"enrolled": 16, "active": 17, "completed": 18, "succeeded": 19}, "personaTagIds": [], "contactCounts": {"active": 0, "enrolled": 0}, "portalId": 8727216, "insertedAt": 1610635850758, "updatedAt": 1611847907579, "contactListIds_enrolled": 16, "contactListIds_active": 17, "contactListIds_completed": 18, "contactListIds_succeeded": 19}, "emitted_at": 1697714264419} {"stream": "workflows", "data": {"migrationStatus": {"portalId": 8727216, "workflowId": 21058122, "migrationStatus": "EXECUTION_MIGRATED", "enrollmentMigrationStatus": "PLATFORM_OWNED", "platformOwnsActions": true, "lastSuccessfulMigrationTimestamp": null, "enrollmentMigrationTimestamp": null, "flowId": 50205036}, "name": "Test Workflow 2", "id": 21058122, "type": "DRIP_DELAY", "enabled": false, "creationSource": {"sourceApplication": {"source": "DIRECT_API"}, "createdAt": 1610635859664}, "updateSource": {"sourceApplication": {"source": "DIRECT_API", "serviceName": "AutomationPlatformService-web_BackfillILSListIds"}, "updatedAt": 1611847907578}, "contactListIds": {"enrolled": 20, "active": 21, "completed": 22, "succeeded": 23}, "personaTagIds": [], "contactCounts": {"active": 0, "enrolled": 0}, "portalId": 8727216, "insertedAt": 1610635859748, "updatedAt": 1611847907578, "contactListIds_enrolled": 20, "contactListIds_active": 21, "contactListIds_completed": 22, "contactListIds_succeeded": 23}, "emitted_at": 1697714264420} @@ -69,4 +69,4 @@ {"stream": "pets", "data": {"id": "5936415312", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:08:50.632000+00:00", "hs_lastmodifieddate": "2023-04-12T17:08:50.632000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5936415312, "hs_object_source": "CRM_UI", "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "pet_name": "Marcos Pet", "pet_type": "Dog"}, "createdAt": "2023-04-12T17:08:50.632Z", "updatedAt": "2023-04-12T17:08:50.632Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-12T17:08:50.632000+00:00", "properties_hs_lastmodifieddate": "2023-04-12T17:08:50.632000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 5936415312, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_pinned_engagement_id": null, "properties_hs_read_only": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_pet_name": "Marcos Pet", "properties_pet_type": "Dog"}, "emitted_at": 1703886126793} {"stream": "pets", "data": {"id": "5938880054", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:53:12.692000+00:00", "hs_lastmodifieddate": "2023-04-12T17:53:12.692000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5938880054, "hs_object_source": "CRM_UI", "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "pet_name": "Integration Test Pet", "pet_type": "Unknown"}, "createdAt": "2023-04-12T17:53:12.692Z", "updatedAt": "2023-04-12T17:53:12.692Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-12T17:53:12.692000+00:00", "properties_hs_lastmodifieddate": "2023-04-12T17:53:12.692000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 5938880054, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_pinned_engagement_id": null, "properties_hs_read_only": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_pet_name": "Integration Test Pet", "properties_pet_type": "Unknown"}, "emitted_at": 1703886126795} {"stream": "contacts_web_analytics", "data": {"objectType": "CONTACT", "objectId": "401", "eventType": "pe8727216_airbyte_contact_custom_event", "occurredAt": "2023-12-01T22:08:25.435Z", "id": "d287cdb7-3e8a-4f4d-92db-486e32f99ad4", "properties_hs_region": "officiis exercitationem modi adipisicing odit Hic", "properties_hs_campaign_id": "libero", "properties_hs_page_url": "Lorem", "properties_hs_element_id": "dolor sit", "properties_hs_browser": "architecto molestias, officiis exercitationem sit", "properties_hs_screen_width": "1531.0", "properties_hs_device_type": "sit adipisicing nobis officiis modi dolor sit", "properties_hs_link_href": "dolor magnam,", "properties_hs_element_class": "exercitationem modi nobis amet odit molestias,", "properties_hs_operating_system": "culpa! ipsum adipisicing consectetur nobis culpa!", "properties_hs_touchpoint_source": "libero modi odit ipsum Lorem accusantium culpa!", "properties_hs_utm_medium": "elit. ipsum officiis molestias, ipsum dolor quas"}, "emitted_at": 1701822848687} -{"stream": "contacts_web_analytics", "data": {"objectType": "CONTACT", "objectId": "401", "eventType": "pe8727216_airbyte_contact_custom_event", "occurredAt": "2023-12-01T22:08:25.723Z", "id": "2f756b9a-a68d-4566-8e63-bc66b9149b41", "properties_hs_page_id": "modi sit", "properties_hs_city": "possimus modi culpa! veniam Lorem odit Lorem quas", "properties_hs_parent_module_id": "reprehenderit exercitationem dolor adipisicing", "properties_hs_user_agent": "possimus reprehenderit architecto odit ipsum, sit", "properties_hs_operating_version": "adipisicing", "properties_hs_element_id": "architecto exercitationem consectetur modi Lorem", "properties_hs_page_content_type": "amet", "properties_hs_screen_height": "4588.0", "properties_hs_operating_system": "reiciendis placeat possimus ipsum, adipisicing", "properties_hs_language": "adipisicing reprehenderit sit ipsum, amet nobis", "properties_hs_region": "placeat accusantium adipisicing culpa! modi quas", "properties_hs_utm_source": "molestias, reprehenderit reprehenderit", "properties_hs_referrer": "possimus consectetur odit sit Lorem nobis culpa!"}, "emitted_at": 1701822848688} \ No newline at end of file +{"stream": "contacts_web_analytics", "data": {"objectType": "CONTACT", "objectId": "401", "eventType": "pe8727216_airbyte_contact_custom_event", "occurredAt": "2023-12-01T22:08:25.723Z", "id": "2f756b9a-a68d-4566-8e63-bc66b9149b41", "properties_hs_page_id": "modi sit", "properties_hs_city": "possimus modi culpa! veniam Lorem odit Lorem quas", "properties_hs_parent_module_id": "reprehenderit exercitationem dolor adipisicing", "properties_hs_user_agent": "possimus reprehenderit architecto odit ipsum, sit", "properties_hs_operating_version": "adipisicing", "properties_hs_element_id": "architecto exercitationem consectetur modi Lorem", "properties_hs_page_content_type": "amet", "properties_hs_screen_height": "4588.0", "properties_hs_operating_system": "reiciendis placeat possimus ipsum, adipisicing", "properties_hs_language": "adipisicing reprehenderit sit ipsum, amet nobis", "properties_hs_region": "placeat accusantium adipisicing culpa! modi quas", "properties_hs_utm_source": "molestias, reprehenderit reprehenderit", "properties_hs_referrer": "possimus consectetur odit sit Lorem nobis culpa!"}, "emitted_at": 1701822848688} diff --git a/airbyte-integrations/connectors/source-hubspot/metadata.yaml b/airbyte-integrations/connectors/source-hubspot/metadata.yaml index 0a6414cf98bd..6438581617b8 100644 --- a/airbyte-integrations/connectors/source-hubspot/metadata.yaml +++ b/airbyte-integrations/connectors/source-hubspot/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c - dockerImageTag: 2.0.2 + dockerImageTag: 3.0.0 dockerRepository: airbyte/source-hubspot documentationUrl: https://docs.airbyte.com/integrations/sources/hubspot githubIssueLabel: source-hubspot @@ -34,6 +34,14 @@ data: This version eliminates the Property History stream in favor of creating 3 different streams, Contacts, Companies, and Deals, which can now all fetch their property history. It will affect only users who use Property History stream, who will need to fix schema conflicts and sync Contacts Property History stream instead of Property History. upgradeDeadline: 2024-01-15 + 3.0.0: + message: >- + This update brings extended schema with data type changes for the Marketing Emails stream. + Users will need to refresh it and reset this stream after upgrading. + upgradeDeadline: 2024-02-12 + scopedImpact: + - scopeType: stream + impactedScopes: ["marketing_emails"] suggestedStreams: streams: - contacts diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json index 4da808123cc1..a81f43a1b423 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json @@ -22,6 +22,12 @@ }, "lastSizeChangeAt": { "type": ["null", "integer"] + }, + "listReferencesCount": { + "type": ["null", "integer"] + }, + "parentFolderId": { + "type": ["null", "integer"] } } }, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json index 43ac501b429f..1e76c445fd7f 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json @@ -109,6 +109,24 @@ "items": { "type": ["null", "integer"] } + }, + "contentIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "quoteIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "marketingEventIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } } } }, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json index c1f20f05459f..48e77f1ef8ed 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json @@ -52,6 +52,20 @@ }, "fieldType": { "type": ["null", "string"] + }, + "validation": { + "type": ["null", "object"], + "properties": { + "blockedEmailDomains": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "useDefaultBlockList": { + "type": ["null", "boolean"] + } + } } } } @@ -169,6 +183,9 @@ "type": ["null", "string"] } } + }, + "cssClass": { + "type": ["null", "string"] } } }, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/marketing_emails.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/marketing_emails.json index aa15243a3daf..f36b58d95e6f 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/marketing_emails.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/marketing_emails.json @@ -33,7 +33,7 @@ "type": ["null", "string"] }, "aifeatures": { - "type": ["null", "string"] + "type": ["null", "object"] }, "allEmailCampaignIds": { "type": ["null", "array"], @@ -215,6 +215,9 @@ "resolvedDomain": { "type": ["null", "string"] }, + "rootMicId": { + "type": ["null", "string"] + }, "selected": { "type": ["null", "integer"] }, @@ -233,6 +236,9 @@ "counters": { "type": ["null", "object"], "properties": { + "sent": { + "type": ["null", "integer"] + }, "open": { "type": ["null", "integer"] }, @@ -248,6 +254,9 @@ "click": { "type": ["null", "integer"] }, + "reply": { + "type": ["null", "integer"] + }, "dropped": { "type": ["null", "integer"] }, @@ -313,6 +322,9 @@ "failedToLoad": { "type": ["null", "boolean"] }, + "qualifierStats": { + "type": ["null", "object"] + }, "ratios": { "type": ["null", "object"], "properties": { @@ -328,6 +340,9 @@ "openratio": { "type": ["null", "number"] }, + "replyratio": { + "type": ["null", "number"] + }, "unsubscribedratio": { "type": ["null", "number"] }, @@ -455,164 +470,61 @@ "flexAreas": { "type": ["null", "object"], "properties": { - "properties": { - "main": { - "type": ["null", "object"], - "properties": { - "boxed": { - "type": ["null", "boolean"] - }, - "isSingleColumnFullWidth": { - "type": ["null", "boolean"] - }, - "sections": { - "type": ["null", "array"], - "items": [ - { - "type": ["null", "object"], - "properties": { - "columns": { - "type": ["null", "array"], - "items": [ - { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "widgets": { - "type": ["null", "array"], - "items": [ - { - "type": ["null", "string"] - } - ] - }, - "width": { - "type": ["null", "integer"] - } - } - } - ] - }, - "id": { - "type": ["null", "string"] - }, - "style": { - "type": ["null", "object"], - "properties": { - "backgroundColor": { - "type": ["null", "string"] - }, - "backgroundType": { - "type": ["null", "string"] - }, - "paddingBottom": { - "type": ["null", "string"] - }, - "paddingTop": { + "main": { + "type": ["null", "object"], + "properties": { + "boxed": { + "type": ["null", "boolean"] + }, + "isSingleColumnFullWidth": { + "type": ["null", "boolean"] + }, + "sections": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "columns": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "widgets": { + "type": ["null", "array"], + "items": { "type": ["null", "string"] } + }, + "width": { + "type": ["null", "integer"] } } } }, - { + "id": { + "type": ["null", "string"] + }, + "style": { "type": ["null", "object"], "properties": { - "columns": { - "type": ["null", "array"], - "items": [ - { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "widgets": { - "type": ["null", "array"], - "items": [ - { - "type": ["null", "string"] - } - ] - }, - "width": { - "type": ["null", "integer"] - } - } - } - ] - }, - "id": { + "backgroundColor": { "type": ["null", "string"] }, - "style": { - "type": ["null", "object"], - "properties": { - "backgroundType": { - "type": ["null", "string"] - }, - "paddingBottom": { - "type": ["null", "string"] - }, - "paddingTop": { - "type": ["null", "string"] - } - } - } - } - }, - { - "type": ["null", "object"], - "properties": { - "columns": { - "type": ["null", "array"], - "items": [ - { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "widgets": { - "type": ["null", "array"], - "items": [ - { - "type": ["null", "string"] - } - ] - }, - "width": { - "type": ["null", "integer"] - } - } - } - ] + "backgroundType": { + "type": ["null", "string"] }, - "id": { + "paddingBottom": { "type": ["null", "string"] }, - "style": { - "type": ["null", "object"], - "properties": { - "backgroundColor": { - "type": ["null", "string"] - }, - "backgroundType": { - "type": ["null", "string"] - }, - "paddingBottom": { - "type": ["null", "string"] - }, - "paddingTop": { - "type": ["null", "string"] - } - } + "paddingTop": { + "type": ["null", "string"] } } } - ] + } } } } @@ -659,14 +571,96 @@ "email_body_padding": { "type": ["null", "string"] }, "email_body_width": { "type": ["null", "string"] }, "heading_one_font": { + "type": ["null", "object"], + "properties": { + "bold": { "type": ["null", "boolean"] }, + "color": { "type": ["null", "string"] }, + "font": { "type": ["null", "string"] }, + "font_style": { "type": ["null", "object"] }, + "italic": { "type": ["null", "boolean"] }, + "size": { "type": ["null", "string"] }, + "underline": { "type": ["null", "boolean"] } + } + }, + "heading_two_font": { + "type": ["null", "object"], "properties": { - "bold": { "type": ["null", "string"] }, + "bold": { "type": ["null", "boolean"] }, "color": { "type": ["null", "string"] }, "font": { "type": ["null", "string"] }, "font_style": { "type": ["null", "object"] }, - "italic": { "type": ["null", "string"] }, + "italic": { "type": ["null", "boolean"] }, "size": { "type": ["null", "string"] }, - "underline": { "type": ["null", "string"] } + "underline": { "type": ["null", "boolean"] } + } + }, + "links_font": { + "type": ["null", "object"], + "properties": { + "bold": { "type": ["null", "boolean"] }, + "color": { "type": ["null", "string"] }, + "font": { "type": ["null", "string"] }, + "font_style": { "type": ["null", "object"] }, + "italic": { "type": ["null", "boolean"] }, + "size": { "type": ["null", "string"] }, + "underline": { "type": ["null", "boolean"] } + } + }, + "primary_accent_color": { + "type": ["null", "string"] + }, + "primary_font": { + "type": ["null", "string"] + }, + "primary_font_color": { + "type": ["null", "string"] + }, + "primary_font_line_height": { + "type": ["null", "string"] + }, + "primary_font_size": { + "type": ["null", "string"] + }, + "secondary_accent_color": { + "type": ["null", "string"] + }, + "secondary_font": { + "type": ["null", "string"] + }, + "secondary_font_color": { + "type": ["null", "string"] + }, + "secondary_font_line_height": { + "type": ["null", "string"] + }, + "secondary_font_size": { + "type": ["null", "string"] + }, + "use_email_client_default_settings": { + "type": ["null", "boolean"] + }, + "user_module_defaults": { + "type": ["null", "object"], + "properties": { + "button_email": { + "type": ["null", "object"], + "properties": { + "background_color": { "type": ["null", "string"] }, + "corner_radius": { "type": ["null", "integer"] }, + "font": { "type": ["null", "string"] }, + "font_color": { "type": ["null", "string"] }, + "font_size": { "type": ["null", "integer"] }, + "font_style": { "type": ["null", "object"] } + } + }, + "email_divider": { + "type": ["null", "object"], + "properties": { + "color": { "type": ["null", "object"] }, + "height": { "type": ["null", "integer"] }, + "line_type": { "type": ["null", "string"] } + } + } } } } diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/workflows.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/workflows.json index 228121336cd7..2c40a25a4630 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/workflows.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/workflows.json @@ -40,6 +40,12 @@ "items": { "type": ["null", "string"] } + }, + "completed": { + "type": ["null", "integer"] + }, + "succeeded": { + "type": ["null", "integer"] } } }, diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py index 6989843ff717..bf043aeea2e3 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py @@ -135,18 +135,18 @@ def test_convert_datetime_to_string(): def test_cast_datetime(common_params, caplog): field_value = pendulum.now() - field_name = "curent_time" + field_name = "current_time" Companies(**common_params)._cast_datetime(field_name, field_value) - expected_warining_message = { + expected_warning_message = { "type": "LOG", "log": { "level": "WARN", - "message": f"Couldn't parse date/datetime string in {field_name}, trying to parse timestamp... Field value: {field_value}. Ex: argument 'input': 'DateTime' object cannot be converted to 'PyString'", + "message": f"Couldn't parse date/datetime string in {field_name}, trying to parse timestamp... Field value: {field_value}. Ex: argument of type 'DateTime' is not iterable", }, } - assert expected_warining_message["log"]["message"] in caplog.text + assert expected_warning_message["log"]["message"] in caplog.text def test_check_connection_backoff_on_limit_reached(requests_mock, config): diff --git a/docs/integrations/sources/hubspot-migrations.md b/docs/integrations/sources/hubspot-migrations.md index 498959ee5884..32c8f6ff9997 100644 --- a/docs/integrations/sources/hubspot-migrations.md +++ b/docs/integrations/sources/hubspot-migrations.md @@ -1,5 +1,45 @@ # HubSpot Migration Guide +## Upgrading to 3.0.0 + +:::note +This change is only breaking if you are syncing the Marketing Emails stream. +::: + +This update brings extended schema with data type changes for the Marketing Emails stream. + +Users should: + - Refresh the source schema for the Marketing Emails stream. + - Reset the stream after upgrading to ensure uninterrupted syncs. + +### Refresh affected schemas and reset data + +1. Select **Connections** in the main nav bar. + 1. Select the connection affected by the update. +2. Select the **Replication** tab. + 1. Select **Refresh source schema**. + 2. Select **OK**. + +:::note +Any detected schema changes will be listed for your review. +::: + +3. Select **Save changes** at the bottom of the page. + 1. Ensure the **Reset affected streams** option is checked. + +:::note +Depending on destination type you may not be prompted to reset your data. +::: + +4. Select **Save connection**. + +:::note +This will reset the data in your destination and initiate a fresh sync. +::: + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset) + + ## Upgrading to 2.0.0 :::note diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index 97d00e2a9b94..31af061d5a1c 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -304,6 +304,7 @@ The connector is restricted by normal HubSpot [rate limitations](https://legacyd | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.0.0 | 2024-01-25 | [34492](https://github.com/airbytehq/airbyte/pull/34492) | Update `marketing_emails` stream schema | | 2.0.2 | 2023-12-15 | [33844](https://github.com/airbytehq/airbyte/pull/33844) | Make property_history PK combined to support Incremental/Deduped sync type | | 2.0.1 | 2023-12-15 | [33527](https://github.com/airbytehq/airbyte/pull/33527) | Make query string calculated correctly for ProertyHistory streams to avoid 414 HTTP Errors | | 2.0.0 | 2023-12-08 | [33266](https://github.com/airbytehq/airbyte/pull/33266) | Added ContactsPropertyHistory, CompaniesPropertyHistory, DealsPropertyHistory streams | From 1e80b8df24397a0fa5faed70d82852ca99067044 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 31 Jan 2024 16:32:50 +0100 Subject: [PATCH 44/96] Publish to pypi batch8 (#34690) --- airbyte-integrations/connectors/source-orbit/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-oura/metadata.yaml | 4 ++++ .../connectors/source-outbrain-amplify/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-outreach/metadata.yaml | 4 ++++ .../connectors/source-pagerduty/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-pardot/metadata.yaml | 4 ++++ .../connectors/source-partnerstack/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-paystack/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-pendo/metadata.yaml | 4 ++++ .../connectors/source-persistiq/metadata.yaml | 4 ++++ .../connectors/source-pexels-api/metadata.yaml | 4 ++++ .../connectors/source-pinterest/metadata.yaml | 4 ++++ .../connectors/source-pivotal-tracker/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-plaid/metadata.yaml | 4 ++++ .../connectors/source-plausible/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-pocket/metadata.yaml | 4 ++++ .../connectors/source-polygon-stock-api/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-posthog/metadata.yaml | 4 ++++ .../connectors/source-postmarkapp/metadata.yaml | 4 ++++ 19 files changed, 76 insertions(+) diff --git a/airbyte-integrations/connectors/source-orbit/metadata.yaml b/airbyte-integrations/connectors/source-orbit/metadata.yaml index 5accdc194140..c4ce50c056f2 100644 --- a/airbyte-integrations/connectors/source-orbit/metadata.yaml +++ b/airbyte-integrations/connectors/source-orbit/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - "*" + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-orbit registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-oura/metadata.yaml b/airbyte-integrations/connectors/source-oura/metadata.yaml index 01992bf8de55..a8279f08a933 100644 --- a/airbyte-integrations/connectors/source-oura/metadata.yaml +++ b/airbyte-integrations/connectors/source-oura/metadata.yaml @@ -8,6 +8,10 @@ data: icon: oura.svg license: MIT name: Oura + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-oura registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/metadata.yaml b/airbyte-integrations/connectors/source-outbrain-amplify/metadata.yaml index 502c3d43ef58..06fd36aa0e7d 100644 --- a/airbyte-integrations/connectors/source-outbrain-amplify/metadata.yaml +++ b/airbyte-integrations/connectors/source-outbrain-amplify/metadata.yaml @@ -1,4 +1,8 @@ data: + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-outbrain-amplify registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-outreach/metadata.yaml b/airbyte-integrations/connectors/source-outreach/metadata.yaml index 30fa65743964..3e5d2a105c5e 100644 --- a/airbyte-integrations/connectors/source-outreach/metadata.yaml +++ b/airbyte-integrations/connectors/source-outreach/metadata.yaml @@ -12,6 +12,10 @@ data: icon: outreach.svg license: MIT name: Outreach + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-outreach registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-pagerduty/metadata.yaml b/airbyte-integrations/connectors/source-pagerduty/metadata.yaml index 0b471649a4a4..1d1bce47b87e 100644 --- a/airbyte-integrations/connectors/source-pagerduty/metadata.yaml +++ b/airbyte-integrations/connectors/source-pagerduty/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.pagerduty.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pagerduty registries: oss: enabled: false diff --git a/airbyte-integrations/connectors/source-pardot/metadata.yaml b/airbyte-integrations/connectors/source-pardot/metadata.yaml index 6a0ab9f2d1ff..6bc3843e5f5c 100644 --- a/airbyte-integrations/connectors/source-pardot/metadata.yaml +++ b/airbyte-integrations/connectors/source-pardot/metadata.yaml @@ -8,6 +8,10 @@ data: icon: salesforcepardot.svg license: MIT name: Pardot + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pardot registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-partnerstack/metadata.yaml b/airbyte-integrations/connectors/source-partnerstack/metadata.yaml index 726c61a4f52d..cf6483ba4899 100644 --- a/airbyte-integrations/connectors/source-partnerstack/metadata.yaml +++ b/airbyte-integrations/connectors/source-partnerstack/metadata.yaml @@ -8,6 +8,10 @@ data: icon: partnerstack.svg license: MIT name: PartnerStack + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-partnerstack registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-paystack/metadata.yaml b/airbyte-integrations/connectors/source-paystack/metadata.yaml index d65d615b940b..203dd02b642c 100644 --- a/airbyte-integrations/connectors/source-paystack/metadata.yaml +++ b/airbyte-integrations/connectors/source-paystack/metadata.yaml @@ -11,6 +11,10 @@ data: icon: paystack.svg license: MIT name: Paystack + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-paystack registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-pendo/metadata.yaml b/airbyte-integrations/connectors/source-pendo/metadata.yaml index c3f0ee28127a..be8343b55fa1 100644 --- a/airbyte-integrations/connectors/source-pendo/metadata.yaml +++ b/airbyte-integrations/connectors/source-pendo/metadata.yaml @@ -8,6 +8,10 @@ data: icon: pendo.svg license: MIT name: Pendo + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pendo registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-persistiq/metadata.yaml b/airbyte-integrations/connectors/source-persistiq/metadata.yaml index 2d06105f0138..90a020fc773b 100644 --- a/airbyte-integrations/connectors/source-persistiq/metadata.yaml +++ b/airbyte-integrations/connectors/source-persistiq/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.persistiq.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-persistiq registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-pexels-api/metadata.yaml b/airbyte-integrations/connectors/source-pexels-api/metadata.yaml index 5a90b9a93769..9e5e8cab8fa3 100644 --- a/airbyte-integrations/connectors/source-pexels-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-pexels-api/metadata.yaml @@ -8,6 +8,10 @@ data: icon: pexels.svg license: MIT name: Pexels API + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pexels-api registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-pinterest/metadata.yaml b/airbyte-integrations/connectors/source-pinterest/metadata.yaml index e496ea4154c6..61d7f60529df 100644 --- a/airbyte-integrations/connectors/source-pinterest/metadata.yaml +++ b/airbyte-integrations/connectors/source-pinterest/metadata.yaml @@ -13,6 +13,10 @@ data: icon: pinterest.svg license: MIT name: Pinterest + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pinterest registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/metadata.yaml b/airbyte-integrations/connectors/source-pivotal-tracker/metadata.yaml index 013eac02808a..a41a1713b016 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/metadata.yaml +++ b/airbyte-integrations/connectors/source-pivotal-tracker/metadata.yaml @@ -8,6 +8,10 @@ data: icon: pivotal-tracker.svg license: MIT name: Pivotal Tracker + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pivotal-tracker registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-plaid/metadata.yaml b/airbyte-integrations/connectors/source-plaid/metadata.yaml index e05ecb3ad445..a8407c8ad2b1 100644 --- a/airbyte-integrations/connectors/source-plaid/metadata.yaml +++ b/airbyte-integrations/connectors/source-plaid/metadata.yaml @@ -8,6 +8,10 @@ data: icon: plaid.svg license: MIT name: Plaid + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-plaid registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-plausible/metadata.yaml b/airbyte-integrations/connectors/source-plausible/metadata.yaml index 042fa88dd49a..086d1c33a3c0 100644 --- a/airbyte-integrations/connectors/source-plausible/metadata.yaml +++ b/airbyte-integrations/connectors/source-plausible/metadata.yaml @@ -8,6 +8,10 @@ data: icon: plausible.svg license: MIT name: Plausible + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-plausible registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-pocket/metadata.yaml b/airbyte-integrations/connectors/source-pocket/metadata.yaml index 145bf1b927e4..219e92e32df2 100644 --- a/airbyte-integrations/connectors/source-pocket/metadata.yaml +++ b/airbyte-integrations/connectors/source-pocket/metadata.yaml @@ -8,6 +8,10 @@ data: icon: pocket.svg license: MIT name: Pocket + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pocket registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-polygon-stock-api/metadata.yaml b/airbyte-integrations/connectors/source-polygon-stock-api/metadata.yaml index 9be804fef482..897591536c31 100644 --- a/airbyte-integrations/connectors/source-polygon-stock-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-polygon-stock-api/metadata.yaml @@ -11,6 +11,10 @@ data: icon: polygon.svg license: MIT name: Polygon Stock API + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-polygon-stock-api registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-posthog/metadata.yaml b/airbyte-integrations/connectors/source-posthog/metadata.yaml index 792e53553194..cfd5c0c74779 100644 --- a/airbyte-integrations/connectors/source-posthog/metadata.yaml +++ b/airbyte-integrations/connectors/source-posthog/metadata.yaml @@ -16,6 +16,10 @@ data: icon: posthog.svg license: MIT name: PostHog + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-posthog registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-postmarkapp/metadata.yaml b/airbyte-integrations/connectors/source-postmarkapp/metadata.yaml index bac99f12668c..58849623dd1c 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/metadata.yaml +++ b/airbyte-integrations/connectors/source-postmarkapp/metadata.yaml @@ -8,6 +8,10 @@ data: icon: postmark.svg license: MIT name: Postmark App + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-postmarkapp registries: cloud: enabled: true From a5d3bad02f9d2909b86773a3ea83a49cb2163559 Mon Sep 17 00:00:00 2001 From: Christo Grabowski <108154848+ChristoGrab@users.noreply.github.com> Date: Wed, 31 Jan 2024 10:47:50 -0500 Subject: [PATCH 45/96] Source Azure Table Storage: CDK Update (#34576) --- .../connectors/source-azure-table/Dockerfile | 2 +- .../acceptance-test-config.yml | 41 ++++--- .../integration_tests/abnormal_state.json | 12 +- .../integration_tests/catalog.json | 21 ---- .../integration_tests/configured_catalog.json | 22 ++-- .../integration_tests/state.json | 5 - .../source-azure-table/metadata.yaml | 2 +- .../source_azure_table/source.py | 6 +- .../source_azure_table/streams.py | 2 +- .../source-azure-table/unit_tests/conftest.py | 43 ++++++++ .../unit_tests/test_azure_table.py | 103 ++++++++++++++++++ .../unit_tests/test_source.py | 31 +----- docs/integrations/sources/azure-table.md | 3 +- 13 files changed, 200 insertions(+), 93 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-azure-table/integration_tests/catalog.json delete mode 100644 airbyte-integrations/connectors/source-azure-table/integration_tests/state.json create mode 100644 airbyte-integrations/connectors/source-azure-table/unit_tests/conftest.py create mode 100644 airbyte-integrations/connectors/source-azure-table/unit_tests/test_azure_table.py diff --git a/airbyte-integrations/connectors/source-azure-table/Dockerfile b/airbyte-integrations/connectors/source-azure-table/Dockerfile index 6b59fb5dbbb7..ce1a741668d0 100644 --- a/airbyte-integrations/connectors/source-azure-table/Dockerfile +++ b/airbyte-integrations/connectors/source-azure-table/Dockerfile @@ -34,5 +34,5 @@ COPY source_azure_table ./source_azure_table ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/source-azure-table diff --git a/airbyte-integrations/connectors/source-azure-table/acceptance-test-config.yml b/airbyte-integrations/connectors/source-azure-table/acceptance-test-config.yml index 89f5b61d61a3..dfb70d970ab8 100644 --- a/airbyte-integrations/connectors/source-azure-table/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-azure-table/acceptance-test-config.yml @@ -1,27 +1,32 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-azure-table:dev -tests: +acceptance_tests: spec: - - spec_path: "source_azure_table/spec.json" + tests: + - spec_path: "source_azure_table/spec.json" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] - validate_schema: False + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + validate_schema: False incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - ignored_fields: - "AirbyteTest": ["record"] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-azure-table/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-azure-table/integration_tests/abnormal_state.json index ac616e99229b..c0596a4d3d31 100644 --- a/airbyte-integrations/connectors/source-azure-table/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-azure-table/integration_tests/abnormal_state.json @@ -1,5 +1,9 @@ -{ - "Test": { - "PartitionKey": "abcd" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "PartitionKey": "999" }, + "stream_descriptor": { "name": "pokemon" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-azure-table/integration_tests/catalog.json b/airbyte-integrations/connectors/source-azure-table/integration_tests/catalog.json deleted file mode 100644 index 40d0d8b72734..000000000000 --- a/airbyte-integrations/connectors/source-azure-table/integration_tests/catalog.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "Test", - "json_schema": { - "properties": { - "PartitionKey": { - "type": "string" - } - } - }, - "supported_sync_modes": ["full_refresh", "incremental"] - }, - "source_defined_cursor": true, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["PartitionKey"] - } - ] -} diff --git a/airbyte-integrations/connectors/source-azure-table/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-azure-table/integration_tests/configured_catalog.json index 40d0d8b72734..bfa35916e08e 100644 --- a/airbyte-integrations/connectors/source-azure-table/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-azure-table/integration_tests/configured_catalog.json @@ -2,20 +2,24 @@ "streams": [ { "stream": { - "name": "Test", - "json_schema": { - "properties": { - "PartitionKey": { - "type": "string" - } - } - }, + "name": "pokemon", + "json_schema": {}, + "source_defined_cursor": true, "supported_sync_modes": ["full_refresh", "incremental"] }, - "source_defined_cursor": true, "sync_mode": "incremental", "destination_sync_mode": "append", "cursor_field": ["PartitionKey"] + }, + { + "stream": { + "name": "campaigns", + "json_schema": {}, + "source_defined_cursor": true, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-azure-table/integration_tests/state.json b/airbyte-integrations/connectors/source-azure-table/integration_tests/state.json deleted file mode 100644 index 3859e6df625f..000000000000 --- a/airbyte-integrations/connectors/source-azure-table/integration_tests/state.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "Test": { - "PartitionKey": "1" - } -} diff --git a/airbyte-integrations/connectors/source-azure-table/metadata.yaml b/airbyte-integrations/connectors/source-azure-table/metadata.yaml index 6fa88db58124..f709d284dcee 100644 --- a/airbyte-integrations/connectors/source-azure-table/metadata.yaml +++ b/airbyte-integrations/connectors/source-azure-table/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: source definitionId: 798ae795-5189-42b6-b64e-3cb91db93338 - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 dockerRepository: airbyte/source-azure-table githubIssueLabel: source-azure-table icon: azureblobstorage.svg diff --git a/airbyte-integrations/connectors/source-azure-table/source_azure_table/source.py b/airbyte-integrations/connectors/source-azure-table/source_azure_table/source.py index e5ef7b3f6967..9a2fcc8f567e 100644 --- a/airbyte-integrations/connectors/source-azure-table/source_azure_table/source.py +++ b/airbyte-integrations/connectors/source-azure-table/source_azure_table/source.py @@ -39,6 +39,7 @@ def get_typed_schema(self) -> object: return { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": True, "properties": {"PartitionKey": {"type": "string"}}, } @@ -50,7 +51,7 @@ def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConn next(tables_iterator) return AirbyteConnectionStatus(status=Status.SUCCEEDED) except StopIteration: - logger.log("No tables found, but credentials are correct.") + logger.info("The credentials you provided are valid, but no tables were found in the Storage Account.") return AirbyteConnectionStatus(status=Status.SUCCEEDED) except Exception as e: return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {str(e)}") @@ -70,8 +71,7 @@ def discover(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteC default_cursor_field=["PartitionKey"], ) streams.append(stream) - logger.info(f"Total {streams.count} streams found.") - + logger.info(f"Total {len(streams)} streams found.") return AirbyteCatalog(streams=streams) def streams(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> List[Stream]: diff --git a/airbyte-integrations/connectors/source-azure-table/source_azure_table/streams.py b/airbyte-integrations/connectors/source-azure-table/source_azure_table/streams.py index 13f68760f540..d4f2669109a2 100644 --- a/airbyte-integrations/connectors/source-azure-table/source_azure_table/streams.py +++ b/airbyte-integrations/connectors/source-azure-table/source_azure_table/streams.py @@ -24,7 +24,7 @@ def name(self): return self.stream_name def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): - return {self.cursor_field[0]: latest_record.record.data.get(self.cursor_field[0])} + return {self.cursor_field[0]: latest_record.data.get(self.cursor_field[0])} def _update_state(self, latest_cursor): self._state = latest_cursor diff --git a/airbyte-integrations/connectors/source-azure-table/unit_tests/conftest.py b/airbyte-integrations/connectors/source-azure-table/unit_tests/conftest.py new file mode 100644 index 000000000000..4b78c28bb43d --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-table/unit_tests/conftest.py @@ -0,0 +1,43 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from unittest import mock + +import pytest +from source_azure_table.azure_table import AzureTableReader +from source_azure_table.source import SourceAzureTable + + +# Fixtures +@pytest.fixture +def config(): + return {"storage_account_name": "dummy-value", "storage_access_key": "dummy-value", "storage_endpoint_suffix": "dummy-value"} + + +@pytest.fixture +def tables(): + table1 = mock.Mock() + table1.name = "AzureTable1" + table2 = mock.Mock() + table2.name = "AzureTable2" + + tables = mock.MagicMock() + tables.__iter__.return_value = [table1, table2] + return tables + + +@pytest.fixture +def source(): + return SourceAzureTable() + + +@pytest.fixture +def logger(): + return logging.getLogger("airbyte") + + +@pytest.fixture +def reader(config, logger): + return AzureTableReader(logger, config) diff --git a/airbyte-integrations/connectors/source-azure-table/unit_tests/test_azure_table.py b/airbyte-integrations/connectors/source-azure-table/unit_tests/test_azure_table.py new file mode 100644 index 000000000000..752e8472480c --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-table/unit_tests/test_azure_table.py @@ -0,0 +1,103 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest + + +def test_get_table_service_client_return(mocker, reader): + """ + Test that the get_table_service_client method returns the expected Table Service Client. + """ + mock_client = "dummy-client" + mocker.patch( + "source_azure_table.azure_table.TableServiceClient.from_connection_string", + return_value=mock_client, + ) + + client = reader.get_table_service_client() + assert client == mock_client + + +def test_get_table_service_client_handles_exception(mocker, reader): + """ + Test that get_table_service_client method handles exceptions correctly. + """ + mocker.patch( + "source_azure_table.azure_table.TableServiceClient.from_connection_string", + side_effect=Exception("Connection error") + ) + + with pytest.raises(Exception) as exc_info: + reader.get_table_service_client() + + assert "Connection error" in str(exc_info.value) + + +def test_get_table_client_return(mocker, reader): + """ + Test that the get_table_client method returns the expected Table Client. + """ + mock_client = "dummy-client" + mocker.patch( + "source_azure_table.azure_table.TableClient.from_connection_string", + return_value=mock_client, + ) + + table = reader.get_table_client("dummy-table") + assert table == mock_client + + +def test_get_table_client_handles_exception(mocker, reader): + """ + Test that get_table_client method handles exceptions correctly. + """ + + # The method throws its own exception for empty table names + with pytest.raises(Exception) as exc_info: + reader.get_table_client("") + assert "table name is not valid." in str(exc_info.value) + + mocker.patch( + "source_azure_table.azure_table.TableClient.from_connection_string", + side_effect=Exception("Connection error") + ) + + with pytest.raises(Exception) as exc_info: + reader.get_table_client("valid_table_name") + assert "Connection error" in str(exc_info.value) + + +def test_get_tables_return(mocker, reader, tables): + """ + Test that the get_tables method returns the expected tables. + """ + mock_client = mocker.MagicMock() + mock_client.list_tables.return_value = tables.__iter__() + mocker.patch( + "azure.data.tables.TableServiceClient.from_connection_string", + return_value=mock_client + ) + + result = reader.get_tables() + result_table_names = [table.name for table in result] + + expected_table_names = ["AzureTable1", "AzureTable2"] + assert result_table_names == expected_table_names + + +def test_get_tables_handles_exception(mocker, reader): + """ + Test that get_tables method handles exceptions correctly. + """ + mock_client = mocker.MagicMock() + mock_client.list_tables.side_effect = Exception("Failed to list tables") + mocker.patch( + "azure.data.tables.TableServiceClient.from_connection_string", + return_value=mock_client + ) + + with pytest.raises(Exception) as exc_info: + reader.get_tables() + + assert "Failed to list tables" in str(exc_info.value) diff --git a/airbyte-integrations/connectors/source-azure-table/unit_tests/test_source.py b/airbyte-integrations/connectors/source-azure-table/unit_tests/test_source.py index bc1ce50f12bc..956375acda0b 100644 --- a/airbyte-integrations/connectors/source-azure-table/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-azure-table/unit_tests/test_source.py @@ -2,38 +2,12 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import logging -from unittest import mock - -import pytest from airbyte_cdk.models import AirbyteCatalog, SyncMode -from source_azure_table.source import SourceAzureTable from source_azure_table.streams import AzureTableStream -source = SourceAzureTable() -logger = logging.getLogger() - - -# Fixtures -@pytest.fixture -def config(): - return {"storage_account_name": "dummy-value", "storage_access_key": "dummy-value", "storage_endpoint_suffix": "dummy-value"} - - -@pytest.fixture -def tables(): - table1 = mock.Mock() - table1.name = "AzureTable1" - table2 = mock.Mock() - table2.name = "AzureTable2" - - tables = mock.MagicMock() - tables.__iter__.return_value = [table1, table2] - return tables - # Tests -def test_discover(mocker, config, tables): +def test_discover(mocker, config, tables, source, logger): mocker.patch( "source_azure_table.azure_table.AzureTableReader.get_tables", return_value=tables, @@ -47,6 +21,7 @@ def test_discover(mocker, config, tables): assert stream.json_schema == { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": True, "properties": {"PartitionKey": {"type": "string"}}, } assert stream.supported_sync_modes == [SyncMode.full_refresh, SyncMode.incremental] @@ -54,7 +29,7 @@ def test_discover(mocker, config, tables): assert stream.default_cursor_field == ["PartitionKey"] -def test_streams(mocker, config, tables): +def test_streams(mocker, config, tables, source, logger): mocker.patch( "source_azure_table.azure_table.AzureTableReader.get_tables", return_value=tables, diff --git a/docs/integrations/sources/azure-table.md b/docs/integrations/sources/azure-table.md index 6ea26ec6c34e..f617018961a6 100644 --- a/docs/integrations/sources/azure-table.md +++ b/docs/integrations/sources/azure-table.md @@ -62,12 +62,11 @@ Visit the [Azure Portal](https://portal.azure.com). Go to your storage account, We recommend creating a restricted key specifically for Airbyte access. This will allow you to control which resources Airbyte should be able to access. However, shared access key authentication is not supported by this connector yet. - ## Changelog | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------ | +| 0.1.4 | 2024-01-26 | [34576](https://github.com/airbytehq/airbyte/pull/34576) | Migrate to per-stream/global state | | 0.1.3 | 2022-08-12 | [15591](https://github.com/airbytehq/airbyte/pull/15591) | Clean instantiation of AirbyteStream | | 0.1.2 | 2021-12-23 | [14212](https://github.com/airbytehq/airbyte/pull/14212) | Adding incremental load capability | | 0.1.1 | 2021-12-23 | [8434](https://github.com/airbytehq/airbyte/pull/8434) | Update fields in source-connectors specifications | - From 2cff69041bfa39cbeab75459eab4ead2d2d44f62 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 31 Jan 2024 17:34:02 +0100 Subject: [PATCH 46/96] Publish to pypi batch9 (#34691) --- .../connectors/source-prestashop/metadata.yaml | 4 ++++ .../connectors/source-primetric/metadata.yaml | 4 ++++ .../connectors/source-public-apis/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-punk-api/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-pypi/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-qualaroo/metadata.yaml | 4 ++++ .../connectors/source-quickbooks/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-railz/metadata.yaml | 4 ++++ .../connectors/source-rd-station-marketing/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-recharge/metadata.yaml | 4 ++++ .../connectors/source-recreation/metadata.yaml | 4 ++++ .../connectors/source-recruitee/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-recurly/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-reply-io/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-retently/metadata.yaml | 4 ++++ .../connectors/source-ringcentral/metadata.yaml | 4 ++++ .../connectors/source-rki-covid/metadata.yaml | 4 ++++ .../connectors/source-rocket-chat/metadata.yaml | 4 ++++ 18 files changed, 72 insertions(+) diff --git a/airbyte-integrations/connectors/source-prestashop/metadata.yaml b/airbyte-integrations/connectors/source-prestashop/metadata.yaml index 184347008685..dbbf6240c187 100644 --- a/airbyte-integrations/connectors/source-prestashop/metadata.yaml +++ b/airbyte-integrations/connectors/source-prestashop/metadata.yaml @@ -15,6 +15,10 @@ data: icon: prestashop.svg license: MIT name: PrestaShop + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-prestashop registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-primetric/metadata.yaml b/airbyte-integrations/connectors/source-primetric/metadata.yaml index 38b37950abad..83bd3b3ecda9 100644 --- a/airbyte-integrations/connectors/source-primetric/metadata.yaml +++ b/airbyte-integrations/connectors/source-primetric/metadata.yaml @@ -8,6 +8,10 @@ data: icon: primetric.svg license: MIT name: Primetric + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-primetric registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-public-apis/metadata.yaml b/airbyte-integrations/connectors/source-public-apis/metadata.yaml index c83aba288893..fe8481eaf1f1 100644 --- a/airbyte-integrations/connectors/source-public-apis/metadata.yaml +++ b/airbyte-integrations/connectors/source-public-apis/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - "*" + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-public-apis registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-punk-api/metadata.yaml b/airbyte-integrations/connectors/source-punk-api/metadata.yaml index 942b617b200f..80d6d03f5a1a 100644 --- a/airbyte-integrations/connectors/source-punk-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-punk-api/metadata.yaml @@ -8,6 +8,10 @@ data: icon: punkapi.svg license: MIT name: Punk API + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-punk-api registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-pypi/metadata.yaml b/airbyte-integrations/connectors/source-pypi/metadata.yaml index 90cd84ad6509..978205dca0d5 100644 --- a/airbyte-integrations/connectors/source-pypi/metadata.yaml +++ b/airbyte-integrations/connectors/source-pypi/metadata.yaml @@ -8,6 +8,10 @@ data: icon: pypi.svg license: MIT name: PyPI + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pypi registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-qualaroo/metadata.yaml b/airbyte-integrations/connectors/source-qualaroo/metadata.yaml index e291b82f879a..f33f9ef7bdec 100644 --- a/airbyte-integrations/connectors/source-qualaroo/metadata.yaml +++ b/airbyte-integrations/connectors/source-qualaroo/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - "*" # Please change to the hostname of the source. + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-qualaroo registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-quickbooks/metadata.yaml b/airbyte-integrations/connectors/source-quickbooks/metadata.yaml index 0961738252b3..c75af443cb2c 100644 --- a/airbyte-integrations/connectors/source-quickbooks/metadata.yaml +++ b/airbyte-integrations/connectors/source-quickbooks/metadata.yaml @@ -13,6 +13,10 @@ data: icon: quickbooks.svg license: MIT name: QuickBooks + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-quickbooks registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-railz/metadata.yaml b/airbyte-integrations/connectors/source-railz/metadata.yaml index ea4aa848d729..eb0870a1779a 100644 --- a/airbyte-integrations/connectors/source-railz/metadata.yaml +++ b/airbyte-integrations/connectors/source-railz/metadata.yaml @@ -8,6 +8,10 @@ data: icon: railz.svg license: MIT name: Railz + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-railz registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/metadata.yaml b/airbyte-integrations/connectors/source-rd-station-marketing/metadata.yaml index 42135ace43a0..dbfc299e01bb 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-rd-station-marketing/metadata.yaml @@ -8,6 +8,10 @@ data: icon: rdstation.svg license: MIT name: RD Station Marketing + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-rd-station-marketing registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-recharge/metadata.yaml b/airbyte-integrations/connectors/source-recharge/metadata.yaml index af32cc639991..c6850d60ea3f 100644 --- a/airbyte-integrations/connectors/source-recharge/metadata.yaml +++ b/airbyte-integrations/connectors/source-recharge/metadata.yaml @@ -13,6 +13,10 @@ data: icon: recharge.svg license: MIT name: Recharge + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-recharge registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-recreation/metadata.yaml b/airbyte-integrations/connectors/source-recreation/metadata.yaml index b217b6b65c04..821554d94e1d 100644 --- a/airbyte-integrations/connectors/source-recreation/metadata.yaml +++ b/airbyte-integrations/connectors/source-recreation/metadata.yaml @@ -8,6 +8,10 @@ data: icon: recreation.svg license: MIT name: Recreation + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-recreation registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-recruitee/metadata.yaml b/airbyte-integrations/connectors/source-recruitee/metadata.yaml index 42407ff998bf..71061b162cbd 100644 --- a/airbyte-integrations/connectors/source-recruitee/metadata.yaml +++ b/airbyte-integrations/connectors/source-recruitee/metadata.yaml @@ -8,6 +8,10 @@ data: icon: recruitee.svg license: MIT name: Recruitee + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-recruitee registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-recurly/metadata.yaml b/airbyte-integrations/connectors/source-recurly/metadata.yaml index d10aee87e084..4d39d72c0b58 100644 --- a/airbyte-integrations/connectors/source-recurly/metadata.yaml +++ b/airbyte-integrations/connectors/source-recurly/metadata.yaml @@ -12,6 +12,10 @@ data: icon: recurly.svg license: MIT name: Recurly + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-recurly registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-reply-io/metadata.yaml b/airbyte-integrations/connectors/source-reply-io/metadata.yaml index 9ad7fcafafd6..d57e6cb62bdf 100644 --- a/airbyte-integrations/connectors/source-reply-io/metadata.yaml +++ b/airbyte-integrations/connectors/source-reply-io/metadata.yaml @@ -8,6 +8,10 @@ data: icon: reply-io.svg license: MIT name: Reply.io + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-reply-io registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-retently/metadata.yaml b/airbyte-integrations/connectors/source-retently/metadata.yaml index 40818fb95bca..77e0ea8d3fd7 100644 --- a/airbyte-integrations/connectors/source-retently/metadata.yaml +++ b/airbyte-integrations/connectors/source-retently/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - "*" # Please change to the hostname of the source. + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-retently registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-ringcentral/metadata.yaml b/airbyte-integrations/connectors/source-ringcentral/metadata.yaml index e92f20f242b9..62c18c12ec9d 100644 --- a/airbyte-integrations/connectors/source-ringcentral/metadata.yaml +++ b/airbyte-integrations/connectors/source-ringcentral/metadata.yaml @@ -8,6 +8,10 @@ data: icon: ringcentral.svg license: MIT name: Ringcentral + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-ringcentral registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-rki-covid/metadata.yaml b/airbyte-integrations/connectors/source-rki-covid/metadata.yaml index fb3acfd90923..d11317580f3d 100644 --- a/airbyte-integrations/connectors/source-rki-covid/metadata.yaml +++ b/airbyte-integrations/connectors/source-rki-covid/metadata.yaml @@ -8,6 +8,10 @@ data: icon: rki.svg license: MIT name: RKI Covid + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-rki-covid registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-rocket-chat/metadata.yaml b/airbyte-integrations/connectors/source-rocket-chat/metadata.yaml index b78e01397580..35229f131695 100644 --- a/airbyte-integrations/connectors/source-rocket-chat/metadata.yaml +++ b/airbyte-integrations/connectors/source-rocket-chat/metadata.yaml @@ -8,6 +8,10 @@ data: icon: rocket-chat.svg license: MIT name: Rocket.chat + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-rocket-chat registries: cloud: enabled: false From 1ba7b213902a7e487c7dda29fa68d5681e1eddc1 Mon Sep 17 00:00:00 2001 From: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Date: Wed, 31 Jan 2024 09:48:12 -0800 Subject: [PATCH 47/96] Build a resume token with a pipeline consisting of selected streams (#34594) Co-authored-by: Akash Kulkarni Co-authored-by: Akash Kulkarni <113392464+akashkulk@users.noreply.github.com> --- .../connectors/source-mongodb-v2/build.gradle | 2 +- .../source-mongodb-v2/metadata.yaml | 2 +- .../mongodb/cdc/MongoDbCdcInitializer.java | 2 +- .../mongodb/cdc/MongoDbCdcProperties.java | 3 + .../cdc/MongoDbDebeziumPropertiesManager.java | 2 + .../mongodb/cdc/MongoDbResumeTokenHelper.java | 19 ++++- .../mongodb/MongoDbSourceAcceptanceTest.java | 8 +- .../cdc/MongoDbCdcInitializerTest.java | 7 ++ .../mongodb/cdc/MongoDbCdcPropertiesTest.java | 5 +- .../cdc/MongoDbCdcTargetPositionTest.java | 74 ++++++++++++++----- .../MongoDbDebeziumPropertiesManagerTest.java | 8 +- .../cdc/MongoDbResumeTokenHelperTest.java | 18 ++++- docs/integrations/sources/mongodb-v2.md | 3 +- 13 files changed, 120 insertions(+), 33 deletions(-) diff --git a/airbyte-integrations/connectors/source-mongodb-v2/build.gradle b/airbyte-integrations/connectors/source-mongodb-v2/build.gradle index e273d8950bdf..43fb1b80b40d 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/build.gradle +++ b/airbyte-integrations/connectors/source-mongodb-v2/build.gradle @@ -5,7 +5,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.16.0' + cdkVersionRequired = '0.16.3' features = ['db-sources'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml b/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml index 469a031351f2..6d4aa4f311c1 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml +++ b/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: source definitionId: b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e - dockerImageTag: 1.2.5 + dockerImageTag: 1.2.6 dockerRepository: airbyte/source-mongodb-v2 documentationUrl: https://docs.airbyte.com/integrations/sources/mongodb-v2 githubIssueLabel: source-mongodb-v2 diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializer.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializer.java index 57059a86ce44..4e6078a6b2f3 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializer.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializer.java @@ -87,7 +87,7 @@ public List> createCdcIterators( final boolean isEnforceSchema = config.getEnforceSchema(); final Properties defaultDebeziumProperties = MongoDbCdcProperties.getDebeziumProperties(); logOplogInfo(mongoClient); - final BsonDocument initialResumeToken = MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient); + final BsonDocument initialResumeToken = MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, databaseName, catalog); final JsonNode initialDebeziumState = mongoDbDebeziumStateUtil.constructInitialDebeziumState(initialResumeToken, mongoClient, databaseName); final MongoDbCdcState cdcState = (stateManager.getCdcState() == null || stateManager.getCdcState().state() == null) diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcProperties.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcProperties.java index e567d190da32..0e559a634c2a 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcProperties.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcProperties.java @@ -20,6 +20,8 @@ public class MongoDbCdcProperties { static final String HEARTBEAT_INTERVAL_KEY = "heartbeat.interval.ms"; static final String SNAPSHOT_MODE_KEY = "snapshot.mode"; static final String SNAPSHOT_MODE_VALUE = "never"; + static final String CAPTURE_SCOPE_KEY = "capture.scope"; + static final String CAPTURE_SCOPE_VALUE = "database"; static final String TOMBSTONE_ON_DELETE_KEY = "tombstones.on.delete"; static final String TOMBSTONE_ON_DELETE_VALUE = Boolean.FALSE.toString(); @@ -33,6 +35,7 @@ public static Properties getDebeziumProperties() { props.setProperty(CONNECTOR_CLASS_KEY, CONNECTOR_CLASS_VALUE); props.setProperty(SNAPSHOT_MODE_KEY, SNAPSHOT_MODE_VALUE); + props.setProperty(CAPTURE_SCOPE_KEY, CAPTURE_SCOPE_VALUE); props.setProperty(CAPTURE_MODE_KEY, CAPTURE_MODE_VALUE); props.setProperty(HEARTBEAT_INTERVAL_KEY, HEARTBEAT_FREQUENCY_MS); props.setProperty(TOMBSTONE_ON_DELETE_KEY, TOMBSTONE_ON_DELETE_VALUE); diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumPropertiesManager.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumPropertiesManager.java index dde6d653802e..c715be6080cd 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumPropertiesManager.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumPropertiesManager.java @@ -31,6 +31,7 @@ public class MongoDbDebeziumPropertiesManager extends DebeziumPropertiesManager static final String COLLECTION_INCLUDE_LIST_KEY = "collection.include.list"; static final String DATABASE_INCLUDE_LIST_KEY = "database.include.list"; + static final String CAPTURE_TARGET_KEY = "capture.target"; static final String DOUBLE_QUOTES_PATTERN = "\""; static final String MONGODB_AUTHSOURCE_KEY = "mongodb.authsource"; static final String MONGODB_CONNECTION_MODE_KEY = "mongodb.connection.mode"; @@ -79,6 +80,7 @@ protected Properties getIncludeConfiguration(final ConfiguredAirbyteCatalog cata // Database/collection selection properties.setProperty(COLLECTION_INCLUDE_LIST_KEY, createCollectionIncludeString(catalog.getStreams())); properties.setProperty(DATABASE_INCLUDE_LIST_KEY, config.get(DATABASE_CONFIGURATION_KEY).asText()); + properties.setProperty(CAPTURE_TARGET_KEY, config.get(DATABASE_CONFIGURATION_KEY).asText()); return properties; } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbResumeTokenHelper.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbResumeTokenHelper.java index de8c4d44a66c..0493efa11cda 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbResumeTokenHelper.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbResumeTokenHelper.java @@ -8,11 +8,18 @@ import com.mongodb.client.ChangeStreamIterable; import com.mongodb.client.MongoChangeStreamCursor; import com.mongodb.client.MongoClient; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.Filters; import com.mongodb.client.model.changestream.ChangeStreamDocument; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.concurrent.TimeUnit; import org.bson.BsonDocument; import org.bson.BsonTimestamp; +import org.bson.conversions.Bson; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,8 +36,16 @@ public class MongoDbResumeTokenHelper { * @param mongoClient The {@link MongoClient} used to query the MongoDB server. * @return The most recent resume token value. */ - public static BsonDocument getMostRecentResumeToken(final MongoClient mongoClient) { - final ChangeStreamIterable eventStream = mongoClient.watch(BsonDocument.class); + public static BsonDocument getMostRecentResumeToken(final MongoClient mongoClient, + final String databaseName, + final ConfiguredAirbyteCatalog catalog) { + final List collectionsList = catalog.getStreams().stream() + .map(s -> s.getStream().getName()) + .toList(); + LOGGER.info("Resume token for db {} with collection filter {}", databaseName, Arrays.toString(collectionsList.toArray())); + final List pipeline = Collections.singletonList(Aggregates.match( + Filters.in("ns.coll", collectionsList))); + final ChangeStreamIterable eventStream = mongoClient.getDatabase(databaseName).watch(pipeline, BsonDocument.class); try (final MongoChangeStreamCursor> eventStreamCursor = eventStream.cursor()) { /* * Must call tryNext before attempting to get the resume token from the cursor directly. Otherwise, diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongoDbSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongoDbSourceAcceptanceTest.java index b82144007b16..fa220e4386db 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongoDbSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongoDbSourceAcceptanceTest.java @@ -522,7 +522,8 @@ void testSyncShouldHandlePurgedLogsGracefully() throws Exception { void testReachedTargetPosition() { final long eventTimestamp = Long.MAX_VALUE; final Integer order = 0; - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, databaseName, getConfiguredCatalog())); final ChangeEventWithMetadata changeEventWithMetadata = mock(ChangeEventWithMetadata.class); when(changeEventWithMetadata.isSnapshotEvent()).thenReturn(true); @@ -549,8 +550,9 @@ void testReachedTargetPosition() { @Test void testIsSameOffset() { - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); - final BsonDocument resumeToken = MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient); + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, databaseName, getConfiguredCatalog())); + final BsonDocument resumeToken = MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, databaseName, getConfiguredCatalog()); final String resumeTokenString = resumeToken.get("_data").asString().getValue(); final String replicaSet = MongoDbDebeziumStateUtil.getReplicaSetName(mongoClient); final Map emptyOffsetA = Map.of(); diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializerTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializerTest.java index 59b62ee0688b..9b87de23c1a3 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializerTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializerTest.java @@ -30,6 +30,8 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.Filters; import com.mongodb.client.model.changestream.ChangeStreamDocument; import com.mongodb.connection.ClusterDescription; import com.mongodb.connection.ClusterType; @@ -58,6 +60,7 @@ import io.airbyte.protocol.models.v0.SyncMode; import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -65,6 +68,7 @@ import org.bson.BsonDocument; import org.bson.BsonString; import org.bson.Document; +import org.bson.conversions.Bson; import org.bson.types.ObjectId; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -106,6 +110,8 @@ class MongoDbCdcInitializerTest { private MongoCursor findCursor; private ChangeStreamIterable changeStreamIterable; private MongoDbCdcConnectorMetadataInjector cdcConnectorMetadataInjector; + private static final List PIPELINE = Collections.singletonList(Aggregates.match( + Filters.in("ns.coll", List.of(COLLECTION)))); @BeforeEach void setUp() { @@ -132,6 +138,7 @@ void setUp() { when(clusterDescription.getServerDescriptions()).thenReturn(List.of(serverDescription)); when(clusterDescription.getType()).thenReturn(ClusterType.REPLICA_SET); when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + when(mongoDatabase.watch(PIPELINE, BsonDocument.class)).thenReturn(changeStreamIterable); when(mongoClient.getDatabase(DATABASE)).thenReturn(mongoDatabase); when(mongoClient.getClusterDescription()).thenReturn(clusterDescription); when(mongoDatabase.getCollection(COLLECTION)).thenReturn(mongoCollection); diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcPropertiesTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcPropertiesTest.java index 0bb2bfba05f3..f6ea739be583 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcPropertiesTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcPropertiesTest.java @@ -6,6 +6,8 @@ import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.CAPTURE_MODE_KEY; import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.CAPTURE_MODE_VALUE; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.CAPTURE_SCOPE_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.CAPTURE_SCOPE_VALUE; import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.CONNECTOR_CLASS_KEY; import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.CONNECTOR_CLASS_VALUE; import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.HEARTBEAT_FREQUENCY_MS; @@ -24,12 +26,13 @@ class MongoDbCdcPropertiesTest { @Test void testDebeziumProperties() { final Properties debeziumProperties = MongoDbCdcProperties.getDebeziumProperties(); - assertEquals(5, debeziumProperties.size()); + assertEquals(6, debeziumProperties.size()); assertEquals(CONNECTOR_CLASS_VALUE, debeziumProperties.get(CONNECTOR_CLASS_KEY)); assertEquals(SNAPSHOT_MODE_VALUE, debeziumProperties.get(SNAPSHOT_MODE_KEY)); assertEquals(CAPTURE_MODE_VALUE, debeziumProperties.get(CAPTURE_MODE_KEY)); assertEquals(HEARTBEAT_FREQUENCY_MS, debeziumProperties.get(HEARTBEAT_INTERVAL_KEY)); assertEquals(TOMBSTONE_ON_DELETE_VALUE, debeziumProperties.get(TOMBSTONE_ON_DELETE_KEY)); + assertEquals(CAPTURE_SCOPE_VALUE, debeziumProperties.get(CAPTURE_SCOPE_KEY)); } } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcTargetPositionTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcTargetPositionTest.java index 9e015d786a65..20cf0b1ef9cb 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcTargetPositionTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcTargetPositionTest.java @@ -10,6 +10,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -17,17 +18,24 @@ import com.mongodb.client.ChangeStreamIterable; import com.mongodb.client.MongoChangeStreamCursor; import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.Filters; import com.mongodb.client.model.changestream.ChangeStreamDocument; import io.airbyte.cdk.integrations.debezium.internals.ChangeEventWithMetadata; import io.airbyte.commons.resources.MoreResources; import io.airbyte.protocol.models.Jsons; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.debezium.connector.mongodb.ResumeTokens; import io.debezium.engine.ChangeEvent; import java.io.IOException; +import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import org.bson.BsonDocument; import org.bson.BsonTimestamp; +import org.bson.conversions.Bson; import org.junit.jupiter.api.Test; class MongoDbCdcTargetPositionTest { @@ -35,6 +43,10 @@ class MongoDbCdcTargetPositionTest { private static final String OBJECT_ID = "64f24244f95155351c4185b1"; private static final String RESUME_TOKEN = "8264BEB9F3000000012B0229296E04"; private static final String OTHER_RESUME_TOKEN = "8264BEB9F3000000012B0229296E05"; + private static final ConfiguredAirbyteCatalog CATALOG = new ConfiguredAirbyteCatalog(); + private static final String DATABASE = "test-database"; + private static final List PIPELINE = Collections.singletonList(Aggregates.match( + Filters.in("ns.coll", Collections.emptyList()))); @Test void testCreateTargetPosition() { @@ -43,12 +55,15 @@ void testCreateTargetPosition() { final MongoChangeStreamCursor> mongoChangeStreamCursor = mock(MongoChangeStreamCursor.class); final MongoClient mongoClient = mock(MongoClient.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + when(mongoClient.getDatabase(anyString())).thenReturn(mongoDatabase); + when(mongoDatabase.watch(PIPELINE, BsonDocument.class)).thenReturn(changeStreamIterable); - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, DATABASE, CATALOG)); assertNotNull(targetPosition); assertEquals(ResumeTokens.getTimestamp(resumeTokenDocument), targetPosition.getResumeTokenTimestamp()); } @@ -61,16 +76,19 @@ void testReachedTargetPosition() throws IOException { final MongoChangeStreamCursor> mongoChangeStreamCursor = mock(MongoChangeStreamCursor.class); final MongoClient mongoClient = mock(MongoClient.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); final ChangeEvent changeEvent = mock(ChangeEvent.class); when(changeEvent.key()).thenReturn("{\"" + ID_FIELD + "\":\"{\\\"" + OBJECT_ID_FIELD + "\\\": \\\"" + OBJECT_ID + "\\\"}\"}"); when(changeEvent.value()).thenReturn(changeEventJson); when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + when(mongoClient.getDatabase(anyString())).thenReturn(mongoDatabase); + when(mongoDatabase.watch(PIPELINE, BsonDocument.class)).thenReturn(changeStreamIterable); final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, DATABASE, CATALOG)); assertTrue(targetPosition.reachedTargetPosition(changeEventWithMetadata)); when(changeEvent.value()).thenReturn(changeEventJson.replaceAll("\"ts_ms\": \\d+,", "\"ts_ms\": 1590221043000,")); @@ -86,16 +104,19 @@ void testReachedTargetPositionSnapshotEvent() throws IOException { final MongoChangeStreamCursor> mongoChangeStreamCursor = mock(MongoChangeStreamCursor.class); final MongoClient mongoClient = mock(MongoClient.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); final ChangeEvent changeEvent = mock(ChangeEvent.class); when(changeEvent.key()).thenReturn("{\"" + ID_FIELD + "\":\"{\\\"" + OBJECT_ID_FIELD + "\\\": \\\"" + OBJECT_ID + "\\\"}\"}"); when(changeEvent.value()).thenReturn(changeEventJson); when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + when(mongoClient.getDatabase(anyString())).thenReturn(mongoDatabase); + when(mongoDatabase.watch(PIPELINE, BsonDocument.class)).thenReturn(changeStreamIterable); final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, DATABASE, CATALOG)); assertFalse(targetPosition.reachedTargetPosition(changeEventWithMetadata)); } @@ -107,16 +128,20 @@ void testReachedTargetPositionSnapshotLastEvent() throws IOException { final MongoChangeStreamCursor> mongoChangeStreamCursor = mock(MongoChangeStreamCursor.class); final MongoClient mongoClient = mock(MongoClient.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); final ChangeEvent changeEvent = mock(ChangeEvent.class); when(changeEvent.key()).thenReturn("{\"" + ID_FIELD + "\":\"{\\\"" + OBJECT_ID_FIELD + "\\\": \\\"" + OBJECT_ID + "\\\"}\"}"); when(changeEvent.value()).thenReturn(changeEventJson); when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(mongoClient.getDatabase(anyString())).thenReturn(mongoDatabase); + when(mongoDatabase.watch(PIPELINE, BsonDocument.class)).thenReturn(changeStreamIterable); when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, DATABASE, CATALOG)); assertTrue(targetPosition.reachedTargetPosition(changeEventWithMetadata)); } @@ -127,12 +152,15 @@ void testReachedTargetPositionFromHeartbeat() { final MongoChangeStreamCursor> mongoChangeStreamCursor = mock(MongoChangeStreamCursor.class); final MongoClient mongoClient = mock(MongoClient.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + when(mongoClient.getDatabase(anyString())).thenReturn(mongoDatabase); + when(mongoDatabase.watch(PIPELINE, BsonDocument.class)).thenReturn(changeStreamIterable); - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, DATABASE, CATALOG)); final BsonTimestamp heartbeatTimestamp = new BsonTimestamp( Long.valueOf(ResumeTokens.getTimestamp(resumeTokenDocument).getTime() + TimeUnit.HOURS.toSeconds(1)).intValue(), 0); @@ -148,12 +176,15 @@ void testIsHeartbeatSupported() { final MongoChangeStreamCursor> mongoChangeStreamCursor = mock(MongoChangeStreamCursor.class); final MongoClient mongoClient = mock(MongoClient.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + when(mongoClient.getDatabase(anyString())).thenReturn(mongoDatabase); + when(mongoDatabase.watch(PIPELINE, BsonDocument.class)).thenReturn(changeStreamIterable); - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, DATABASE, CATALOG)); assertTrue(targetPosition.isHeartbeatSupported()); } @@ -166,12 +197,15 @@ void testExtractPositionFromHeartbeatOffset() { final MongoChangeStreamCursor> mongoChangeStreamCursor = mock(MongoChangeStreamCursor.class); final MongoClient mongoClient = mock(MongoClient.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + when(mongoClient.getDatabase(anyString())).thenReturn(mongoDatabase); + when(mongoDatabase.watch(PIPELINE, BsonDocument.class)).thenReturn(changeStreamIterable); - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, DATABASE, CATALOG)); final Map sourceOffset = Map.of( MongoDbDebeziumConstants.ChangeEvent.SOURCE_SECONDS, resumeTokenTimestamp.getTime(), @@ -189,6 +223,7 @@ void testIsEventAheadOfOffset() throws IOException { final MongoChangeStreamCursor> mongoChangeStreamCursor = mock(MongoChangeStreamCursor.class); final MongoClient mongoClient = mock(MongoClient.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); final String changeEventJson = MoreResources.readResource("mongodb/change_event.json"); final ChangeEvent changeEvent = mock(ChangeEvent.class); @@ -196,13 +231,15 @@ void testIsEventAheadOfOffset() throws IOException { when(changeEvent.value()).thenReturn(changeEventJson); when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + when(mongoClient.getDatabase(anyString())).thenReturn(mongoDatabase); + when(mongoDatabase.watch(PIPELINE, BsonDocument.class)).thenReturn(changeStreamIterable); final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); final Map offset = Jsons.object(MongoDbDebeziumStateUtil.formatState(null, null, RESUME_TOKEN), new TypeReference<>() {}); - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, DATABASE, CATALOG)); final boolean result = targetPosition.isEventAheadOffset(offset, changeEventWithMetadata); assertTrue(result); } @@ -214,10 +251,12 @@ void testIsSameOffset() { final MongoChangeStreamCursor> mongoChangeStreamCursor = mock(MongoChangeStreamCursor.class); final MongoClient mongoClient = mock(MongoClient.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + when(mongoClient.getDatabase(anyString())).thenReturn(mongoDatabase); + when(mongoDatabase.watch(PIPELINE, BsonDocument.class)).thenReturn(changeStreamIterable); final Map offsetA = Jsons.object(MongoDbDebeziumStateUtil.formatState(null, null, RESUME_TOKEN), new TypeReference<>() {}); @@ -226,7 +265,8 @@ void testIsSameOffset() { final Map offsetC = Jsons.object(MongoDbDebeziumStateUtil.formatState(null, null, OTHER_RESUME_TOKEN), new TypeReference<>() {}); - final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + final MongoDbCdcTargetPosition targetPosition = + new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, DATABASE, CATALOG)); assertTrue(targetPosition.isSameOffset(offsetA, offsetA)); assertTrue(targetPosition.isSameOffset(offsetA, offsetB)); diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumPropertiesManagerTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumPropertiesManagerTest.java index e2f87486fb0c..c635a8e37dde 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumPropertiesManagerTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbDebeziumPropertiesManagerTest.java @@ -67,7 +67,7 @@ void testDebeziumProperties() { final var debeziumPropertiesManager = new MongoDbDebeziumPropertiesManager(cdcProperties, config, catalog); final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(offsetManager); - assertEquals(19 + cdcProperties.size(), debeziumProperties.size()); + assertEquals(20 + cdcProperties.size(), debeziumProperties.size()); assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(NAME_KEY)); assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(TOPIC_PREFIX_KEY)); assertEquals(EXPECTED_CONNECTION_STRING, debeziumProperties.get(MONGODB_CONNECTION_STRING_KEY)); @@ -97,7 +97,7 @@ void testDebeziumPropertiesConnectionStringCredentialsPlaceholder() { final var debeziumPropertiesManager = new MongoDbDebeziumPropertiesManager(cdcProperties, config, catalog); final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(offsetManager); - assertEquals(19 + cdcProperties.size(), debeziumProperties.size()); + assertEquals(20 + cdcProperties.size(), debeziumProperties.size()); assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(NAME_KEY)); assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(TOPIC_PREFIX_KEY)); assertEquals(EXPECTED_CONNECTION_STRING, debeziumProperties.get(MONGODB_CONNECTION_STRING_KEY)); @@ -126,7 +126,7 @@ void testDebeziumPropertiesQuotedConnectionString() { final var debeziumPropertiesManager = new MongoDbDebeziumPropertiesManager(cdcProperties, config, catalog); final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(offsetManager); - assertEquals(19 + cdcProperties.size(), debeziumProperties.size()); + assertEquals(20 + cdcProperties.size(), debeziumProperties.size()); assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(NAME_KEY)); assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(TOPIC_PREFIX_KEY)); assertEquals(EXPECTED_CONNECTION_STRING, debeziumProperties.get(MONGODB_CONNECTION_STRING_KEY)); @@ -155,7 +155,7 @@ void testDebeziumPropertiesNoCredentials() { final var debeziumPropertiesManager = new MongoDbDebeziumPropertiesManager(cdcProperties, config, catalog); final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(offsetManager); - assertEquals(16 + cdcProperties.size(), debeziumProperties.size()); + assertEquals(17 + cdcProperties.size(), debeziumProperties.size()); assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(NAME_KEY)); assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(TOPIC_PREFIX_KEY)); assertEquals(EXPECTED_CONNECTION_STRING, debeziumProperties.get(MONGODB_CONNECTION_STRING_KEY)); diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbResumeTokenHelperTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbResumeTokenHelperTest.java index dfddbfeef6be..39fd65567eac 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbResumeTokenHelperTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbResumeTokenHelperTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -14,32 +15,45 @@ import com.mongodb.client.ChangeStreamIterable; import com.mongodb.client.MongoChangeStreamCursor; import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.Filters; import com.mongodb.client.model.changestream.ChangeStreamDocument; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.debezium.connector.mongodb.ResumeTokens; import java.io.IOException; +import java.util.Collections; +import java.util.List; import java.util.concurrent.TimeUnit; import org.bson.BsonDocument; import org.bson.BsonTimestamp; +import org.bson.conversions.Bson; import org.junit.jupiter.api.Test; class MongoDbResumeTokenHelperTest { + private static final String DATABASE = "test-database"; + @Test void testRetrievingResumeToken() { final String resumeToken = "8264BEB9F3000000012B0229296E04"; final BsonDocument resumeTokenDocument = ResumeTokens.fromData(resumeToken); final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); final MongoChangeStreamCursor> mongoChangeStreamCursor = mock(MongoChangeStreamCursor.class); final MongoClient mongoClient = mock(MongoClient.class); when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + when(mongoClient.getDatabase(anyString())).thenReturn(mongoDatabase); + final List pipeline = Collections.singletonList(Aggregates.match( + Filters.in("ns.coll", Collections.emptyList()))); + when(mongoDatabase.watch(pipeline, BsonDocument.class)).thenReturn(changeStreamIterable); - final BsonDocument actualResumeToken = MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient); + final BsonDocument actualResumeToken = MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient, DATABASE, new ConfiguredAirbyteCatalog()); assertEquals(resumeTokenDocument, actualResumeToken); } diff --git a/docs/integrations/sources/mongodb-v2.md b/docs/integrations/sources/mongodb-v2.md index dfd5de71bb39..a406039a1e2c 100644 --- a/docs/integrations/sources/mongodb-v2.md +++ b/docs/integrations/sources/mongodb-v2.md @@ -214,7 +214,8 @@ For more information regarding configuration parameters, please see [MongoDb Doc | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------| -| 1.2.5 | 2024-01-29 | [34573](https://github.com/airbytehq/airbyte/pull/34573) | Allow resuming an initial snapshot when Id type is not of default ObjectId . | +| 1.2.6 | 2024-01-31 | [34594](https://github.com/airbytehq/airbyte/pull/34594) | Scope initial resume token to streams of interest. | +| 1.2.5 | 2024-01-29 | [34641](https://github.com/airbytehq/airbyte/pull/34641) | Allow resuming an initial snapshot when Id type is not of default ObjectId . | | 1.2.4 | 2024-01-26 | [34573](https://github.com/airbytehq/airbyte/pull/34573) | Adopt CDK v0.16.0. | | 1.2.3 | 2024-01-18 | [34364](https://github.com/airbytehq/airbyte/pull/34364) | Add additional logging for resume token + reduce discovery size to 10. | | 1.2.2 | 2024-01-16 | [34314](https://github.com/airbytehq/airbyte/pull/34314) | Reduce minimum document discovery size to 100. | From c6bc976c861e4f55fc70e202a0ba2a60662be793 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 31 Jan 2024 18:51:35 +0100 Subject: [PATCH 48/96] Publish to pypi batch10 (#34692) --- airbyte-integrations/connectors/source-rss/metadata.yaml | 4 ++++ .../connectors/source-salesloft/metadata.yaml | 4 ++++ .../connectors/source-sap-fieldglass/metadata.yaml | 4 ++++ .../connectors/source-scaffold-source-http/metadata.yaml | 4 ++++ .../connectors/source-scaffold-source-python/metadata.yaml | 4 ++++ .../connectors/source-search-metrics/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-secoda/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-sendgrid/metadata.yaml | 4 ++++ .../connectors/source-sendinblue/metadata.yaml | 4 ++++ .../connectors/source-senseforce/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-sentry/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-serpstat/metadata.yaml | 4 ++++ .../connectors/source-sftp-bulk/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-shortio/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-smaily/metadata.yaml | 4 ++++ .../connectors/source-smartengage/metadata.yaml | 4 ++++ .../connectors/source-snapchat-marketing/metadata.yaml | 4 ++++ .../connectors/source-sonar-cloud/metadata.yaml | 4 ++++ .../connectors/source-spacex-api/metadata.yaml | 4 ++++ airbyte-integrations/connectors/source-square/metadata.yaml | 4 ++++ 20 files changed, 80 insertions(+) diff --git a/airbyte-integrations/connectors/source-rss/metadata.yaml b/airbyte-integrations/connectors/source-rss/metadata.yaml index 50a3f9531e7b..8e7d7b8fdc91 100644 --- a/airbyte-integrations/connectors/source-rss/metadata.yaml +++ b/airbyte-integrations/connectors/source-rss/metadata.yaml @@ -8,6 +8,10 @@ data: icon: rss.svg license: MIT name: RSS + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-rss registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-salesloft/metadata.yaml b/airbyte-integrations/connectors/source-salesloft/metadata.yaml index 7fdf8beebb6b..a5c2cc049e9e 100644 --- a/airbyte-integrations/connectors/source-salesloft/metadata.yaml +++ b/airbyte-integrations/connectors/source-salesloft/metadata.yaml @@ -15,6 +15,10 @@ data: icon: salesloft.svg license: MIT name: SalesLoft + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-salesloft registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-sap-fieldglass/metadata.yaml b/airbyte-integrations/connectors/source-sap-fieldglass/metadata.yaml index 2fdfa8eeeba4..6362a99fd348 100644 --- a/airbyte-integrations/connectors/source-sap-fieldglass/metadata.yaml +++ b/airbyte-integrations/connectors/source-sap-fieldglass/metadata.yaml @@ -8,6 +8,10 @@ data: icon: sapfieldglass.svg license: MIT name: SAP Fieldglass + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-sap-fieldglass registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/metadata.yaml b/airbyte-integrations/connectors/source-scaffold-source-http/metadata.yaml index 3e50afa2130e..c56a4810bde9 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/metadata.yaml +++ b/airbyte-integrations/connectors/source-scaffold-source-http/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - TODO # Please change to the hostname of the source. + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-scaffold-source-http registries: oss: enabled: false diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/metadata.yaml b/airbyte-integrations/connectors/source-scaffold-source-python/metadata.yaml index 2a740dccdc56..24ab5f9c09c1 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/metadata.yaml +++ b/airbyte-integrations/connectors/source-scaffold-source-python/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - TODO # Please change to the hostname of the source. + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-scaffold-source-python registries: oss: enabled: false diff --git a/airbyte-integrations/connectors/source-search-metrics/metadata.yaml b/airbyte-integrations/connectors/source-search-metrics/metadata.yaml index ee0f969317bd..347c807bf39f 100644 --- a/airbyte-integrations/connectors/source-search-metrics/metadata.yaml +++ b/airbyte-integrations/connectors/source-search-metrics/metadata.yaml @@ -8,6 +8,10 @@ data: icon: searchmetrics.svg license: MIT name: SearchMetrics + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-search-metrics registries: cloud: enabled: false diff --git a/airbyte-integrations/connectors/source-secoda/metadata.yaml b/airbyte-integrations/connectors/source-secoda/metadata.yaml index 03c428b40206..cc7a8383aefe 100644 --- a/airbyte-integrations/connectors/source-secoda/metadata.yaml +++ b/airbyte-integrations/connectors/source-secoda/metadata.yaml @@ -8,6 +8,10 @@ data: icon: secoda.svg license: MIT name: Secoda + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-secoda registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-sendgrid/metadata.yaml b/airbyte-integrations/connectors/source-sendgrid/metadata.yaml index fae7388cf808..8466dc5aa32e 100644 --- a/airbyte-integrations/connectors/source-sendgrid/metadata.yaml +++ b/airbyte-integrations/connectors/source-sendgrid/metadata.yaml @@ -17,6 +17,10 @@ data: icon: sendgrid.svg license: MIT name: Sendgrid + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-sendgrid registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-sendinblue/metadata.yaml b/airbyte-integrations/connectors/source-sendinblue/metadata.yaml index 4cebe426e4a3..0318419f4d85 100644 --- a/airbyte-integrations/connectors/source-sendinblue/metadata.yaml +++ b/airbyte-integrations/connectors/source-sendinblue/metadata.yaml @@ -12,6 +12,10 @@ data: icon: sendinblue.svg license: MIT name: Sendinblue + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-sendinblue registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-senseforce/metadata.yaml b/airbyte-integrations/connectors/source-senseforce/metadata.yaml index 3c9417880e60..c5d5d8dd7695 100644 --- a/airbyte-integrations/connectors/source-senseforce/metadata.yaml +++ b/airbyte-integrations/connectors/source-senseforce/metadata.yaml @@ -12,6 +12,10 @@ data: icon: senseforce.svg license: MIT name: Senseforce + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-senseforce registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-sentry/metadata.yaml b/airbyte-integrations/connectors/source-sentry/metadata.yaml index 724e98d0823f..2b77883ec609 100644 --- a/airbyte-integrations/connectors/source-sentry/metadata.yaml +++ b/airbyte-integrations/connectors/source-sentry/metadata.yaml @@ -18,6 +18,10 @@ data: license: MIT maxSecondsBetweenMessages: 64800 name: Sentry + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-sentry registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-serpstat/metadata.yaml b/airbyte-integrations/connectors/source-serpstat/metadata.yaml index fb9d7de24737..849ad93a11da 100644 --- a/airbyte-integrations/connectors/source-serpstat/metadata.yaml +++ b/airbyte-integrations/connectors/source-serpstat/metadata.yaml @@ -2,6 +2,10 @@ data: allowedHosts: hosts: - api.serpstat.com + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-serpstat registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-sftp-bulk/metadata.yaml b/airbyte-integrations/connectors/source-sftp-bulk/metadata.yaml index 9adbfaedfa33..6bddeaa476e5 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/metadata.yaml +++ b/airbyte-integrations/connectors/source-sftp-bulk/metadata.yaml @@ -12,6 +12,10 @@ data: icon: sftp.svg license: MIT name: SFTP Bulk + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-sftp-bulk registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-shortio/metadata.yaml b/airbyte-integrations/connectors/source-shortio/metadata.yaml index 6eced88a6f02..b14f05840fa2 100644 --- a/airbyte-integrations/connectors/source-shortio/metadata.yaml +++ b/airbyte-integrations/connectors/source-shortio/metadata.yaml @@ -3,6 +3,10 @@ data: hosts: - https://api.short.io - https://api-v2.short.cm + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-shortio registries: oss: enabled: true diff --git a/airbyte-integrations/connectors/source-smaily/metadata.yaml b/airbyte-integrations/connectors/source-smaily/metadata.yaml index f2105bbe100b..408ce79e8f32 100644 --- a/airbyte-integrations/connectors/source-smaily/metadata.yaml +++ b/airbyte-integrations/connectors/source-smaily/metadata.yaml @@ -8,6 +8,10 @@ data: icon: smaily.svg license: MIT name: Smaily + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-smaily registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-smartengage/metadata.yaml b/airbyte-integrations/connectors/source-smartengage/metadata.yaml index 25485ef405e6..4fd15c960d1a 100644 --- a/airbyte-integrations/connectors/source-smartengage/metadata.yaml +++ b/airbyte-integrations/connectors/source-smartengage/metadata.yaml @@ -8,6 +8,10 @@ data: icon: smartengage.svg license: MIT name: SmartEngage + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-smartengage registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/metadata.yaml b/airbyte-integrations/connectors/source-snapchat-marketing/metadata.yaml index 060d8a709d1e..36c060f33fdf 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-snapchat-marketing/metadata.yaml @@ -12,6 +12,10 @@ data: icon: snapchat.svg license: MIT name: Snapchat Marketing + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-snapchat-marketing registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-sonar-cloud/metadata.yaml b/airbyte-integrations/connectors/source-sonar-cloud/metadata.yaml index 1d1cf8e9b488..cc224211c7bd 100644 --- a/airbyte-integrations/connectors/source-sonar-cloud/metadata.yaml +++ b/airbyte-integrations/connectors/source-sonar-cloud/metadata.yaml @@ -11,6 +11,10 @@ data: icon: sonarcloud.svg license: MIT name: Sonar Cloud + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-sonar-cloud registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-spacex-api/metadata.yaml b/airbyte-integrations/connectors/source-spacex-api/metadata.yaml index 1da6b6776c15..6f1677c520b9 100644 --- a/airbyte-integrations/connectors/source-spacex-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-spacex-api/metadata.yaml @@ -8,6 +8,10 @@ data: icon: spacex.svg license: MIT name: SpaceX API + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-spacex-api registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-square/metadata.yaml b/airbyte-integrations/connectors/source-square/metadata.yaml index 71b100e1f7c1..7e85289fa520 100644 --- a/airbyte-integrations/connectors/source-square/metadata.yaml +++ b/airbyte-integrations/connectors/source-square/metadata.yaml @@ -16,6 +16,10 @@ data: icon: square.svg license: MIT name: Square + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-square registries: cloud: enabled: true From a2e4026d873ffcf95d89a0f7a84a7fcb2173a292 Mon Sep 17 00:00:00 2001 From: Gireesh Sreepathi Date: Wed, 31 Jan 2024 09:58:18 -0800 Subject: [PATCH 49/96] Destination Postgres: Upgrade CDK with fixed dependency and unpin cloud (#34683) --- .../destination-postgres-strict-encrypt/build.gradle | 2 +- .../destination-postgres-strict-encrypt/metadata.yaml | 3 +-- .../connectors/destination-postgres/build.gradle | 2 +- .../connectors/destination-postgres/metadata.yaml | 3 +-- docs/integrations/destinations/postgres.md | 1 + 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle index ffb340acfbb1..6d672fa3616e 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.16.2' + cdkVersionRequired = '0.16.3' features = [ 'db-sources', // required for tests 'db-destinations' diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml index 6403177cee3a..2b99bfe3d7e4 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 25c5221d-dce2-4163-ade9-739ef790f503 - dockerImageTag: 0.6.1 + dockerImageTag: 0.6.2 dockerRepository: airbyte/destination-postgres-strict-encrypt documentationUrl: https://docs.airbyte.com/integrations/destinations/postgres githubIssueLabel: destination-postgres @@ -16,7 +16,6 @@ data: registries: cloud: enabled: false - dockerImageTag: 0.6.0 oss: enabled: false releaseStage: alpha diff --git a/airbyte-integrations/connectors/destination-postgres/build.gradle b/airbyte-integrations/connectors/destination-postgres/build.gradle index 35b66019d54d..6b49a55578df 100644 --- a/airbyte-integrations/connectors/destination-postgres/build.gradle +++ b/airbyte-integrations/connectors/destination-postgres/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.16.2' + cdkVersionRequired = '0.16.3' features = [ 'db-sources', // required for tests 'db-destinations', diff --git a/airbyte-integrations/connectors/destination-postgres/metadata.yaml b/airbyte-integrations/connectors/destination-postgres/metadata.yaml index 730c511df3b8..99b6d29ef6b5 100644 --- a/airbyte-integrations/connectors/destination-postgres/metadata.yaml +++ b/airbyte-integrations/connectors/destination-postgres/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 25c5221d-dce2-4163-ade9-739ef790f503 - dockerImageTag: 0.6.1 + dockerImageTag: 0.6.2 dockerRepository: airbyte/destination-postgres documentationUrl: https://docs.airbyte.com/integrations/destinations/postgres githubIssueLabel: destination-postgres @@ -19,7 +19,6 @@ data: registries: cloud: dockerRepository: airbyte/destination-postgres-strict-encrypt - dockerImageTag: 0.6.0 enabled: true oss: enabled: true diff --git a/docs/integrations/destinations/postgres.md b/docs/integrations/destinations/postgres.md index a546308aef91..9552b38a3e84 100644 --- a/docs/integrations/destinations/postgres.md +++ b/docs/integrations/destinations/postgres.md @@ -170,6 +170,7 @@ Now that you have set up the Postgres destination connector, check out the follo | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------| +| 0.6.2 | 2024-01-30 | [34683](https://github.com/airbytehq/airbyte/pull/34683) | CDK Upgrade 0.16.3; Fix dependency mismatches in slf4j lib | | 0.6.1 | 2024-01-29 | [34630](https://github.com/airbytehq/airbyte/pull/34630) | CDK Upgrade; Use lowercase raw table in T+D queries. | | 0.6.0 | 2024-01-19 | [34372](https://github.com/airbytehq/airbyte/pull/34372) | Add dv2 flag in spec | | 0.5.5 | 2024-01-18 | [34236](https://github.com/airbytehq/airbyte/pull/34236) | Upgrade CDK to 0.13.1; Add indexes in raw table for query optimization | From 2c861063870394206a41301b97e4588071508e4b Mon Sep 17 00:00:00 2001 From: Artem Inzhyyants <36314070+artem1205@users.noreply.github.com> Date: Wed, 31 Jan 2024 19:23:21 +0100 Subject: [PATCH 50/96] CAT: fix NoAdditionalPropertiesValidator (#34709) --- .../bases/connector-acceptance-test/CHANGELOG.md | 3 +++ .../connector_acceptance_test/utils/asserts.py | 2 +- .../bases/connector-acceptance-test/pyproject.toml | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md b/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md index bf133bde47a8..b7e87e84ed86 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md +++ b/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 3.3.3 +Аix `NoAdditionalPropertiesValidator` if no type found in `items` + ## 3.3.2 Fix TestBasicRead.test_read.validate_schema: set `additionalProperties` to False recursively for objects diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/asserts.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/asserts.py index cb4fe51154b6..8362aac10053 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/asserts.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/asserts.py @@ -48,7 +48,7 @@ def add_properties(properties): elif "type" in prop_value and "array" in prop_value["type"]: if ( prop_value.get("items") - and "object" in prop_value.get("items", {}).get("type") + and "object" in prop_value.get("items", {}).get("type", []) and len(prop_value.get("items", {}).get("properties", [])) ): prop_value["items"]["additionalProperties"] = False diff --git a/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml b/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml index 728c9112ceff..e38d6f7939ba 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml +++ b/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "connector-acceptance-test" -version = "3.3.1" +version = "3.3.3" description = "Contains acceptance tests for connectors." authors = ["Airbyte "] license = "MIT" From b81aa62ccc987fef24c28492e3aba6d6edb8d9b8 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Thu, 1 Feb 2024 14:23:04 -0800 Subject: [PATCH 51/96] source recharge for some reason --- .../connectors/source-recharge/main.py | 9 +++++++-- .../connectors/source-recharge/setup.py | 19 +------------------ 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/main.py b/airbyte-integrations/connectors/source-recharge/main.py index d8ccf40b711e..c61ef445b68d 100644 --- a/airbyte-integrations/connectors/source-recharge/main.py +++ b/airbyte-integrations/connectors/source-recharge/main.py @@ -2,7 +2,12 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from source_recharge.run import run + +import sys + +from airbyte_cdk.entrypoint import launch +from source_recharge import SourceRecharge if __name__ == "__main__": - run() + source = SourceRecharge() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-recharge/setup.py b/airbyte-integrations/connectors/source-recharge/setup.py index 3e80432c9c57..bb091a439b28 100644 --- a/airbyte-integrations/connectors/source-recharge/setup.py +++ b/airbyte-integrations/connectors/source-recharge/setup.py @@ -16,30 +16,13 @@ ] setup( - entry_points={ - "console_scripts": [ - "source-recharge=source_recharge.run:run", - ], - }, name="source_recharge", description="Source implementation for Recharge.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={ - "": [ - # Include yaml files in the package (if any) - "*.yml", - "*.yaml", - # Include all json files in the package, up to 4 levels deep - "*.json", - "*/*.json", - "*/*/*.json", - "*/*/*/*.json", - "*/*/*/*/*.json", - ] - }, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, From 61d96d113c6542fcf119485035e99ff3298eec06 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Thu, 1 Feb 2024 14:28:10 -0800 Subject: [PATCH 52/96] delete file --- .../source-recharge/source_recharge/run.py | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-recharge/source_recharge/run.py diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/run.py b/airbyte-integrations/connectors/source-recharge/source_recharge/run.py deleted file mode 100644 index 2a56566c41c2..000000000000 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/run.py +++ /dev/null @@ -1,14 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import sys - -from airbyte_cdk.entrypoint import launch -from source_recharge import SourceRecharge - - -def run(): - source = SourceRecharge() - launch(source, sys.argv[1:]) From 79d6db06c0fc053241c0f2d45e9bcb312d9a2074 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Thu, 1 Feb 2024 17:21:34 -0800 Subject: [PATCH 53/96] passing tests --- .../destination/StandardNameTransformer.java | 1 + .../clickhouse/ClickhouseDestinationTest.java | 23 ++++++++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StandardNameTransformer.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StandardNameTransformer.java index a0bb39cc5d25..8601f70c49e6 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StandardNameTransformer.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StandardNameTransformer.java @@ -31,6 +31,7 @@ public String getNamespace(final String namespace) { } @Override + @Deprecated public String getRawTableName(final String streamName) { return convertStreamName("_airbyte_raw_" + streamName); } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java index 0b05cb932a8e..6b38b89d79ea 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java @@ -13,11 +13,13 @@ import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer; import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteMessage; @@ -26,6 +28,7 @@ import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.CatalogHelpers; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.Comparator; import java.util.List; @@ -95,22 +98,26 @@ static void cleanUp() { @Test void sanityTest() throws Exception { final Destination dest = new ClickhouseDestination(); - final AirbyteMessageConsumer consumer = dest.getConsumer(config, catalog, - Destination::defaultOutputRecordCollector); + DestinationConfig.initialize(config, dest.isV2Destination()); + final SerializedAirbyteMessageConsumer consumer = dest.getSerializedMessageConsumer(config, catalog, + Destination::defaultOutputRecordCollector); final List expectedRecords = generateRecords(10); consumer.start(); expectedRecords.forEach(m -> { try { - consumer.accept(m); + final var strMessage = Jsons.jsonNode(m).toString(); + consumer.accept(strMessage, strMessage.getBytes(StandardCharsets.UTF_8).length); } catch (final Exception e) { throw new RuntimeException(e); } }); - consumer.accept(new AirbyteMessage() + final var abMessage = Jsons.jsonNode(new AirbyteMessage() .withType(Type.STATE) .withState(new AirbyteStateMessage() - .withData(Jsons.jsonNode(ImmutableMap.of(DB_NAME + "." + STREAM_NAME, 10))))); + .withData(Jsons.jsonNode(ImmutableMap.of(DB_NAME + "." + STREAM_NAME, 10))) + )).toString(); + consumer.accept(abMessage, abMessage.getBytes(StandardCharsets.UTF_8).length); consumer.close(); final JdbcDatabase database = new DefaultJdbcDatabase( @@ -126,8 +133,8 @@ void sanityTest() throws Exception { final List actualRecords = database.bufferedResultSetQuery( connection -> connection.createStatement().executeQuery( - String.format("SELECT * FROM %s.%s;", DB_NAME, - namingResolver.getRawTableName(STREAM_NAME))), + String.format("SELECT * FROM %s.%s;", "airbyte_internal", + StreamId.concatenateRawTableName(DB_NAME, STREAM_NAME))), JdbcUtils.getDefaultSourceOperations()::rowToJson); assertEquals( From a59b052a3a0a8316430aeab2146445a6dd8f8239 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Thu, 1 Feb 2024 17:25:09 -0800 Subject: [PATCH 54/96] formatting --- .../destination/clickhouse/ClickhouseDestinationTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java index 6b38b89d79ea..e414e428e63d 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java @@ -100,7 +100,7 @@ void sanityTest() throws Exception { final Destination dest = new ClickhouseDestination(); DestinationConfig.initialize(config, dest.isV2Destination()); final SerializedAirbyteMessageConsumer consumer = dest.getSerializedMessageConsumer(config, catalog, - Destination::defaultOutputRecordCollector); + Destination::defaultOutputRecordCollector); final List expectedRecords = generateRecords(10); consumer.start(); @@ -115,8 +115,8 @@ void sanityTest() throws Exception { final var abMessage = Jsons.jsonNode(new AirbyteMessage() .withType(Type.STATE) .withState(new AirbyteStateMessage() - .withData(Jsons.jsonNode(ImmutableMap.of(DB_NAME + "." + STREAM_NAME, 10))) - )).toString(); + .withData(Jsons.jsonNode(ImmutableMap.of(DB_NAME + "." + STREAM_NAME, 10))))) + .toString(); consumer.accept(abMessage, abMessage.getBytes(StandardCharsets.UTF_8).length); consumer.close(); @@ -134,7 +134,7 @@ void sanityTest() throws Exception { final List actualRecords = database.bufferedResultSetQuery( connection -> connection.createStatement().executeQuery( String.format("SELECT * FROM %s.%s;", "airbyte_internal", - StreamId.concatenateRawTableName(DB_NAME, STREAM_NAME))), + StreamId.concatenateRawTableName(DB_NAME, STREAM_NAME))), JdbcUtils.getDefaultSourceOperations()::rowToJson); assertEquals( From d5c1cca2028fdf6b0f32c3a79aa1fdc89cc25cac Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Fri, 2 Feb 2024 11:13:56 -0800 Subject: [PATCH 55/96] PR Feedback non gradle --- .../jdbc/AbstractJdbcDestination.java | 5 +-- .../typing_deduping/RawOnlySqlGenerator.kt | 24 ++++++++------ .../typing_deduping/FutureUtils.java | 31 +++++++++++++------ 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java index 89b837408e55..2ad41ebc86bb 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java @@ -6,6 +6,7 @@ import static io.airbyte.cdk.integrations.base.errors.messages.ErrorMessage.getErrorMessage; import static io.airbyte.cdk.integrations.util.ConfiguredCatalogUtilKt.addDefaultNamespaceToStreams; +import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.countOfTypingDedupingThreads; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; @@ -323,10 +324,10 @@ private TyperDeduper getV2TyperDeduper(final JsonNode config, final ConfiguredAi final TyperDeduper typerDeduper; if (disableTypeDedupe) { typerDeduper = new NoOpTyperDeduperWithV1V2Migrations<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator, - 8); + countOfTypingDedupingThreads()); } else { typerDeduper = - new DefaultTyperDeduper<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator, 8); + new DefaultTyperDeduper<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator, countOfTypingDedupingThreads()); } return typerDeduper; } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt index 7a8eb52499e6..17447dbf0833 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt @@ -11,50 +11,56 @@ import org.jooq.Field import org.jooq.SQLDialect import java.util.* +/** + * Some Destinations do not support Typing and Deduping but have the updated raw table format + * SqlGenerator implementations are only for "final" tables and are a required input for + * TyperDeduper classes. This implementation appeases that requirement but does not implement + * any "final" table operations. + */ class RawOnlySqlGenerator(private val namingTransformer: NamingConventionTransformer) : JdbcSqlGenerator(namingTransformer) { override fun getStructType(): DataType<*>? { - throw NotImplementedError("This Destination Does not support final tables") + throw NotImplementedError("This Destination does not support final tables") } override fun getArrayType(): DataType<*>? { - throw NotImplementedError("This Destination Does not support final tables") + throw NotImplementedError("This Destination does not support final tables") } override fun getWidestType(): DataType<*>? { - throw NotImplementedError("This Destination Does not support final tables") + throw NotImplementedError("This Destination does not support final tables") } override fun getDialect(): SQLDialect? { - throw NotImplementedError("This Destination Does not support final tables") + throw NotImplementedError("This Destination does not support final tables") } override fun extractRawDataFields( columns: LinkedHashMap, useExpensiveSaferCasting: Boolean ): List>? { - throw NotImplementedError("This Destination Does not support final tables") + throw NotImplementedError("This Destination does not support final tables") } override fun buildAirbyteMetaColumn(columns: LinkedHashMap): Field<*>? { - throw NotImplementedError("This Destination Does not support final tables") + throw NotImplementedError("This Destination does not support final tables") } override fun cdcDeletedAtNotNullCondition(): Condition? { - throw NotImplementedError("This Destination Does not support final tables") + throw NotImplementedError("This Destination does not support final tables") } override fun getRowNumber( primaryKey: List, cursorField: Optional ): Field? { - throw NotImplementedError("This Destination Does not support final tables") + throw NotImplementedError("This Destination does not support final tables") } override fun existingSchemaMatchesStreamConfig( stream: StreamConfig, existingTable: TableDefinition ): Boolean { - throw NotImplementedError("This Destination Does not support final tables") + throw NotImplementedError("This Destination does not support final tables") } } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/FutureUtils.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/FutureUtils.java index 349437e4acec..9e05ec6643bc 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/FutureUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/FutureUtils.java @@ -12,29 +12,40 @@ public class FutureUtils { + public static final int DEFAULT_TD_THREAD_COUNT = 8; + + /** - * Allow for configuring the number of typing and deduping threads via an enviornment variable in - * the destination container. + * Allow for configuring the number of typing and deduping threads via an environment variable in the destination container. * * @return the number of threads to use in the typing and deduping pool */ public static int countOfTypingDedupingThreads(final int defaultThreads) { return Optional.ofNullable(System.getenv("TD_THREADS")) - .map(Integer::valueOf) - .orElse(defaultThreads); + .map(Integer::valueOf) + .orElse(defaultThreads); + } + + /** + * Allow for configuring the number of typing and deduping threads via an environment variable in the destination container. + * + * @return the number of threads to use in the typing and deduping pool + */ + public static int countOfTypingDedupingThreads() { + return countOfTypingDedupingThreads(DEFAULT_TD_THREAD_COUNT); } /** - * Log all exceptions from a list of futures, and rethrow the first exception if there is one. This - * mimics the behavior of running the futures in serial, where the first failure + * Log all exceptions from a list of futures, and rethrow the first exception if there is one. This mimics the behavior of running the futures in + * serial, where the first failure */ public static void reduceExceptions(final Collection>> potentialExceptions, final String initialMessage) throws Exception { final List exceptions = potentialExceptions.stream() - .map(CompletableFuture::join) - .filter(Optional::isPresent) - .map(Optional::get) - .toList(); + .map(CompletableFuture::join) + .filter(Optional::isPresent) + .map(Optional::get) + .toList(); ConnectorExceptionUtil.logAllAndThrowFirst(initialMessage, exceptions); } From eeae2247d1015b86a8501bedde487824380c169a Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Fri, 2 Feb 2024 11:29:57 -0800 Subject: [PATCH 56/96] PR Feedback non gradle - 2 --- .../typing_deduping/NoOpRawTableTDLock.kt | 9 ++------- .../NoOpTyperDeduperWithV1V2Migrations.java | 2 +- .../typing_deduping/TyperDeduperUtil.kt | 14 ++++---------- .../clickhouse/ClickhouseSqlOperations.java | 16 ++++++++-------- 4 files changed, 15 insertions(+), 26 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpRawTableTDLock.kt b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpRawTableTDLock.kt index 3cc88d70fdd5..9c26e4d605b8 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpRawTableTDLock.kt +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpRawTableTDLock.kt @@ -9,14 +9,9 @@ class NoOpRawTableTDLock: Lock { override fun lockInterruptibly() {} - override fun tryLock(): Boolean { - // To mimic NoOp behavior always return true that lock is acquired - return true - } + override fun tryLock() = true - override fun tryLock(time: Long, unit: TimeUnit): Boolean { - return tryLock() - } + override fun tryLock(time: Long, unit: TimeUnit) = tryLock() override fun unlock() {} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java index 703b8446711a..218de3632de8 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java @@ -57,7 +57,7 @@ public NoOpTyperDeduperWithV1V2Migrations(final SqlGenerator>> prepareTablesTasks = new HashSet<>(); for (final StreamConfig stream : parsedCatalog.streams()) { diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduperUtil.kt b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduperUtil.kt index f1ed3d1ba38a..c8020d561826 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduperUtil.kt +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduperUtil.kt @@ -1,21 +1,15 @@ package io.airbyte.integrations.base.destination.typing_deduping -import com.google.common.collect.Streams -import java.util.* /** * Extracts all the "raw" and "final" schemas identified in the [parsedCatalog] and ensures they * exist in the Destination Database. */ fun prepareAllSchemas(parsedCatalog: ParsedCatalog, sqlGenerator: SqlGenerator, destinationHandler: DestinationHandler) { - val rawSchema = - parsedCatalog.streams.stream().map { stream: StreamConfig -> stream.id.rawNamespace } - val finalSchema = - parsedCatalog.streams.stream().map { stream: StreamConfig -> stream.id.finalNamespace } - val createAllSchemasSql = Streams.concat(rawSchema, finalSchema) - .filter { obj: String? -> Objects.nonNull(obj) } - .distinct() - .map { schema: String? -> sqlGenerator.createSchema(schema) } + val rawSchema = parsedCatalog.streams.mapNotNull { it.id.rawNamespace } + val finalSchema = parsedCatalog.streams.mapNotNull { it.id.finalNamespace } + val createAllSchemasSql = rawSchema.union(finalSchema) + .map { sqlGenerator.createSchema(it) } .toList() destinationHandler.execute(Sql.concat(createAllSchemasSql)) } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java index 80f13c47c7a2..8bfd8442984b 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java @@ -37,14 +37,14 @@ public boolean isSchemaRequired() { public String createTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { return String.format( """ - CREATE TABLE IF NOT EXISTS %s.%s ( - %s String, - %s String, - %s DateTime64(3, 'GMT') DEFAULT now(), - %s DateTime64(3, 'GMT') NULL, - PRIMARY KEY(%s) - ) - ENGINE = MergeTree; + CREATE TABLE IF NOT EXISTS %s.%s ( + %s String, + %s String, + %s DateTime64(3, 'GMT') DEFAULT now(), + %s DateTime64(3, 'GMT') NULL, + PRIMARY KEY(%s) + ) + ENGINE = MergeTree; """, schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, From b7c4b2d9d6502c7740982a5cb51b72a4afa65321 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Fri, 2 Feb 2024 14:52:21 -0800 Subject: [PATCH 57/96] unneccessary class --- .../base/destination/typing_deduping/LoggingUtil.kt | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/LoggingUtil.kt diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/LoggingUtil.kt b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/LoggingUtil.kt deleted file mode 100644 index ed3552b0ff70..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/LoggingUtil.kt +++ /dev/null @@ -1,11 +0,0 @@ -package io.airbyte.integrations.base.destination.typing_deduping - -import org.slf4j.Logger -import org.slf4j.LoggerFactory - -/** - * Extends all classes to lazily initialize a Slf4j [Logger] - */ -fun R.logger(): Lazy { - return lazy { LoggerFactory.getLogger(this.javaClass) } -} From 64486852193228d095d0b3bfdfbcb0f1442a49c4 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Fri, 2 Feb 2024 16:25:09 -0800 Subject: [PATCH 58/96] change jvm target --- airbyte-cdk/java/airbyte-cdk/core/build.gradle | 15 ++++++++++++--- .../java/airbyte-cdk/db-destinations/build.gradle | 14 ++++++++++++-- .../java/airbyte-cdk/typing-deduping/build.gradle | 14 ++++++++++++-- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/core/build.gradle b/airbyte-cdk/java/airbyte-cdk/core/build.gradle index f32d8ab53f45..9bf7fdabb322 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/core/build.gradle @@ -105,8 +105,17 @@ dependencies { testImplementation 'org.apache.commons:commons-lang3:3.11' testImplementation 'org.xerial.snappy:snappy-java:1.1.8.4' testImplementation 'org.mockito:mockito-core:4.6.1' - implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8" } -compileKotlin {} -compileTestKotlin {} +compileKotlin { + compilerOptions { + jvmTarget = "17" + languageVersion = "1.9" + } +} +compileTestKotlin { + compilerOptions { + jvmTarget = "17" + languageVersion = "1.9" + } +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle index b606d3b7cd1c..178ab248db8f 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle @@ -95,5 +95,15 @@ dependencies { testImplementation libs.junit.jupiter.system.stubs } -compileKotlin {} -compileTestKotlin {} +compileKotlin { + compilerOptions { + jvmTarget = "17" + languageVersion = "1.9" + } +} +compileTestKotlin { + compilerOptions { + jvmTarget = "17" + languageVersion = "1.9" + } +} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle b/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle index 4160813789d6..dbc79f699f55 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle @@ -26,5 +26,15 @@ java { } } -compileKotlin {} -compileTestKotlin {} +compileKotlin { + compilerOptions { + jvmTarget = "17" + languageVersion = "1.9" + } +} +compileTestKotlin { + compilerOptions { + jvmTarget = "17" + languageVersion = "1.9" + } +} From 05f3b7cd5b13d2923c9217457b293315f37964c4 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Fri, 2 Feb 2024 18:19:30 -0800 Subject: [PATCH 59/96] gradle changes --- airbyte-cdk/java/airbyte-cdk/build.gradle | 21 +++++++++++++++++++ .../java/airbyte-cdk/core/build.gradle | 17 --------------- .../airbyte-cdk/db-destinations/build.gradle | 17 --------------- .../airbyte-cdk/typing-deduping/build.gradle | 14 ------------- 4 files changed, 21 insertions(+), 48 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/build.gradle b/airbyte-cdk/java/airbyte-cdk/build.gradle index 1523d78df648..51f098f3b860 100644 --- a/airbyte-cdk/java/airbyte-cdk/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/build.gradle @@ -1,3 +1,9 @@ +import org.jetbrains.kotlin.gradle.dsl.JvmTarget +import org.jetbrains.kotlin.gradle.dsl.KotlinVersion +plugins { + id 'org.jetbrains.kotlin.jvm' version '1.9.22' +} + allprojects { apply plugin: 'java-library' apply plugin: 'maven-publish' @@ -5,6 +11,7 @@ allprojects { apply plugin: 'airbyte-integration-test-java' apply plugin: 'airbyte-performance-test-java' apply plugin: 'java-test-fixtures' + apply plugin: 'org.jetbrains.kotlin.jvm' group 'io.airbyte.cdk' version = getCdkTargetVersion() @@ -41,6 +48,19 @@ subprojects { subproject -> } } } + + compileKotlin { + compilerOptions { + jvmTarget = JvmTarget.JVM_17 + languageVersion = KotlinVersion.KOTLIN_1_9 + } + } + compileTestKotlin { + compilerOptions { + jvmTarget = JvmTarget.JVM_17 + languageVersion = KotlinVersion.KOTLIN_1_9 + } + } } project.configurations { @@ -118,3 +138,4 @@ cleanLocalCache.configure { dependsOn tasks.named('clean') dependsOn subprojects.collect { it.tasks.named('clean') } } + diff --git a/airbyte-cdk/java/airbyte-cdk/core/build.gradle b/airbyte-cdk/java/airbyte-cdk/core/build.gradle index 9bf7fdabb322..1a5922d4172e 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/core/build.gradle @@ -1,7 +1,3 @@ -plugins { - id 'org.jetbrains.kotlin.jvm' version '1.9.22' -} - java { compileJava { options.compilerArgs += "-Xlint:-deprecation,-try,-rawtypes,-overloads,-cast,-unchecked" @@ -106,16 +102,3 @@ dependencies { testImplementation 'org.xerial.snappy:snappy-java:1.1.8.4' testImplementation 'org.mockito:mockito-core:4.6.1' } - -compileKotlin { - compilerOptions { - jvmTarget = "17" - languageVersion = "1.9" - } -} -compileTestKotlin { - compilerOptions { - jvmTarget = "17" - languageVersion = "1.9" - } -} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle index 178ab248db8f..64319a920171 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle @@ -1,7 +1,3 @@ -plugins { - id "org.jetbrains.kotlin.jvm" version "1.9.22" -} - java { compileJava { options.compilerArgs += "-Xlint:-deprecation" @@ -94,16 +90,3 @@ dependencies { testImplementation libs.junit.jupiter.system.stubs } - -compileKotlin { - compilerOptions { - jvmTarget = "17" - languageVersion = "1.9" - } -} -compileTestKotlin { - compilerOptions { - jvmTarget = "17" - languageVersion = "1.9" - } -} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle b/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle index dbc79f699f55..fc3ba8f061e8 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle @@ -1,6 +1,5 @@ plugins { id 'java-library' - id 'org.jetbrains.kotlin.jvm' version '1.9.22' } dependencies { @@ -25,16 +24,3 @@ java { options.compilerArgs.remove("-Werror") } } - -compileKotlin { - compilerOptions { - jvmTarget = "17" - languageVersion = "1.9" - } -} -compileTestKotlin { - compilerOptions { - jvmTarget = "17" - languageVersion = "1.9" - } -} From 86bbd1204a61529d45dc54db38adc0d100775636 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Mon, 5 Feb 2024 13:06:11 -0800 Subject: [PATCH 60/96] clean up thread count access --- .../jdbc/AbstractJdbcDestination.java | 6 ++---- .../typing_deduping/DefaultTyperDeduper.java | 11 +++++------ .../destination/typing_deduping/FutureUtils.java | 16 +++------------- .../NoOpTyperDeduperWithV1V2Migrations.java | 9 ++++----- 4 files changed, 14 insertions(+), 28 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java index 2ad41ebc86bb..f43e1abdea05 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java @@ -6,7 +6,6 @@ import static io.airbyte.cdk.integrations.base.errors.messages.ErrorMessage.getErrorMessage; import static io.airbyte.cdk.integrations.util.ConfiguredCatalogUtilKt.addDefaultNamespaceToStreams; -import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.countOfTypingDedupingThreads; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; @@ -323,11 +322,10 @@ private TyperDeduper getV2TyperDeduper(final JsonNode config, final ConfiguredAi final boolean disableTypeDedupe = !config.has(DISABLE_TYPE_DEDUPE) || config.get(DISABLE_TYPE_DEDUPE).asBoolean(false); final TyperDeduper typerDeduper; if (disableTypeDedupe) { - typerDeduper = new NoOpTyperDeduperWithV1V2Migrations<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator, - countOfTypingDedupingThreads()); + typerDeduper = new NoOpTyperDeduperWithV1V2Migrations<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator); } else { typerDeduper = - new DefaultTyperDeduper<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator, countOfTypingDedupingThreads()); + new DefaultTyperDeduper<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator); } return typerDeduper; } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java index 2678500a271b..3a6a166a0b22 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java @@ -5,7 +5,7 @@ package io.airbyte.integrations.base.destination.typing_deduping; import static io.airbyte.cdk.integrations.base.IntegrationRunner.TYPE_AND_DEDUPE_THREAD_NAME; -import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.countOfTypingDedupingThreads; +import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.getCountOfTypeAndDedupeThreads; import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.reduceExceptions; import static io.airbyte.integrations.base.destination.typing_deduping.TyperDeduperUtilKt.prepareAllSchemas; import static java.util.Collections.singleton; @@ -78,8 +78,7 @@ public DefaultTyperDeduper(final SqlGenerator sqlGenerat final DestinationHandler destinationHandler, final ParsedCatalog parsedCatalog, final DestinationV1V2Migrator v1V2Migrator, - final V2TableMigrator v2TableMigrator, - final int defaultThreadCount) { + final V2TableMigrator v2TableMigrator) { this.sqlGenerator = sqlGenerator; this.destinationHandler = destinationHandler; this.parsedCatalog = parsedCatalog; @@ -89,8 +88,8 @@ public DefaultTyperDeduper(final SqlGenerator sqlGenerat this.streamsWithSuccessfulSetup = ConcurrentHashMap.newKeySet(parsedCatalog.streams().size()); this.tdLocks = new ConcurrentHashMap<>(); this.internalTdLocks = new ConcurrentHashMap<>(); - this.executorService = Executors.newFixedThreadPool(countOfTypingDedupingThreads(defaultThreadCount), - new BasicThreadFactory.Builder().namingPattern(TYPE_AND_DEDUPE_THREAD_NAME).build()); + this.executorService = Executors.newFixedThreadPool(getCountOfTypeAndDedupeThreads(), + new BasicThreadFactory.Builder().namingPattern(TYPE_AND_DEDUPE_THREAD_NAME).build()); } public DefaultTyperDeduper( @@ -99,7 +98,7 @@ public DefaultTyperDeduper( final ParsedCatalog parsedCatalog, final DestinationV1V2Migrator v1V2Migrator, final int defaultThreadCount) { - this(sqlGenerator, destinationHandler, parsedCatalog, v1V2Migrator, new NoopV2TableMigrator(), defaultThreadCount); + this(sqlGenerator, destinationHandler, parsedCatalog, v1V2Migrator, new NoopV2TableMigrator()); } private void prepareSchemas(final ParsedCatalog parsedCatalog) throws Exception { diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/FutureUtils.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/FutureUtils.java index 9e05ec6643bc..c2fe1d5895c1 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/FutureUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/FutureUtils.java @@ -12,27 +12,17 @@ public class FutureUtils { - public static final int DEFAULT_TD_THREAD_COUNT = 8; - + private static final int DEFAULT_TD_THREAD_COUNT = 8; /** * Allow for configuring the number of typing and deduping threads via an environment variable in the destination container. * * @return the number of threads to use in the typing and deduping pool */ - public static int countOfTypingDedupingThreads(final int defaultThreads) { + public static int getCountOfTypeAndDedupeThreads() { return Optional.ofNullable(System.getenv("TD_THREADS")) .map(Integer::valueOf) - .orElse(defaultThreads); - } - - /** - * Allow for configuring the number of typing and deduping threads via an environment variable in the destination container. - * - * @return the number of threads to use in the typing and deduping pool - */ - public static int countOfTypingDedupingThreads() { - return countOfTypingDedupingThreads(DEFAULT_TD_THREAD_COUNT); + .orElse(DEFAULT_TD_THREAD_COUNT); } /** diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java index 218de3632de8..581fee4adbf4 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java @@ -5,7 +5,7 @@ package io.airbyte.integrations.base.destination.typing_deduping; import static io.airbyte.cdk.integrations.base.IntegrationRunner.TYPE_AND_DEDUPE_THREAD_NAME; -import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.countOfTypingDedupingThreads; +import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.getCountOfTypeAndDedupeThreads; import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.reduceExceptions; import static io.airbyte.integrations.base.destination.typing_deduping.TyperDeduperUtilKt.prepareAllSchemas; @@ -43,15 +43,14 @@ public NoOpTyperDeduperWithV1V2Migrations(final SqlGenerator destinationHandler, final ParsedCatalog parsedCatalog, final DestinationV1V2Migrator v1V2Migrator, - final V2TableMigrator v2TableMigrator, - final int defaultThreadCount) { + final V2TableMigrator v2TableMigrator) { this.sqlGenerator = sqlGenerator; this.destinationHandler = destinationHandler; this.parsedCatalog = parsedCatalog; this.v1V2Migrator = v1V2Migrator; this.v2TableMigrator = v2TableMigrator; - this.executorService = Executors.newFixedThreadPool(countOfTypingDedupingThreads(defaultThreadCount), - new BasicThreadFactory.Builder().namingPattern(TYPE_AND_DEDUPE_THREAD_NAME).build()); + this.executorService = Executors.newFixedThreadPool(getCountOfTypeAndDedupeThreads(), + new BasicThreadFactory.Builder().namingPattern(TYPE_AND_DEDUPE_THREAD_NAME).build()); } @Override From 1e654f80bd6e72bc78e80ff68d83830ad46e8f5e Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Mon, 5 Feb 2024 13:10:52 -0800 Subject: [PATCH 61/96] formatting --- airbyte-cdk/java/airbyte-cdk/build.gradle | 1 - .../typing_deduping/DefaultTyperDeduper.java | 2 +- .../typing_deduping/FutureUtils.java | 19 ++++++++++--------- .../NoOpTyperDeduperWithV1V2Migrations.java | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/build.gradle b/airbyte-cdk/java/airbyte-cdk/build.gradle index 51f098f3b860..7696b46dd69a 100644 --- a/airbyte-cdk/java/airbyte-cdk/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/build.gradle @@ -138,4 +138,3 @@ cleanLocalCache.configure { dependsOn tasks.named('clean') dependsOn subprojects.collect { it.tasks.named('clean') } } - diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java index 3a6a166a0b22..764f888c4e16 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java @@ -89,7 +89,7 @@ public DefaultTyperDeduper(final SqlGenerator sqlGenerat this.tdLocks = new ConcurrentHashMap<>(); this.internalTdLocks = new ConcurrentHashMap<>(); this.executorService = Executors.newFixedThreadPool(getCountOfTypeAndDedupeThreads(), - new BasicThreadFactory.Builder().namingPattern(TYPE_AND_DEDUPE_THREAD_NAME).build()); + new BasicThreadFactory.Builder().namingPattern(TYPE_AND_DEDUPE_THREAD_NAME).build()); } public DefaultTyperDeduper( diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/FutureUtils.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/FutureUtils.java index c2fe1d5895c1..3319af8297a0 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/FutureUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/FutureUtils.java @@ -15,27 +15,28 @@ public class FutureUtils { private static final int DEFAULT_TD_THREAD_COUNT = 8; /** - * Allow for configuring the number of typing and deduping threads via an environment variable in the destination container. + * Allow for configuring the number of typing and deduping threads via an environment variable in + * the destination container. * * @return the number of threads to use in the typing and deduping pool */ public static int getCountOfTypeAndDedupeThreads() { return Optional.ofNullable(System.getenv("TD_THREADS")) - .map(Integer::valueOf) - .orElse(DEFAULT_TD_THREAD_COUNT); + .map(Integer::valueOf) + .orElse(DEFAULT_TD_THREAD_COUNT); } /** - * Log all exceptions from a list of futures, and rethrow the first exception if there is one. This mimics the behavior of running the futures in - * serial, where the first failure + * Log all exceptions from a list of futures, and rethrow the first exception if there is one. This + * mimics the behavior of running the futures in serial, where the first failure */ public static void reduceExceptions(final Collection>> potentialExceptions, final String initialMessage) throws Exception { final List exceptions = potentialExceptions.stream() - .map(CompletableFuture::join) - .filter(Optional::isPresent) - .map(Optional::get) - .toList(); + .map(CompletableFuture::join) + .filter(Optional::isPresent) + .map(Optional::get) + .toList(); ConnectorExceptionUtil.logAllAndThrowFirst(initialMessage, exceptions); } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java index 581fee4adbf4..1fb3faf59def 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java @@ -50,7 +50,7 @@ public NoOpTyperDeduperWithV1V2Migrations(final SqlGenerator Date: Mon, 5 Feb 2024 13:44:48 -0800 Subject: [PATCH 62/96] update yaml and markdown --- .../destination-clickhouse/metadata.yaml | 1 + .../destinations/clickhouse-migrations.md | 32 +++++++++++++++++++ docs/integrations/destinations/clickhouse.md | 3 +- 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 docs/integrations/destinations/clickhouse-migrations.md diff --git a/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml b/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml index e0775a1f3351..b6cc2b944a79 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml +++ b/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml @@ -17,6 +17,7 @@ data: releases: breakingChanges: 1.0.0: + upgradeDeadline: "2024-03-15" message: > This version removes the option to use "normalization" with clickhouse. It also changes the schema and database of Airbyte's "raw" tables to be compatible with the new diff --git a/docs/integrations/destinations/clickhouse-migrations.md b/docs/integrations/destinations/clickhouse-migrations.md new file mode 100644 index 000000000000..8386e911b804 --- /dev/null +++ b/docs/integrations/destinations/clickhouse-migrations.md @@ -0,0 +1,32 @@ +# Clickhouse Migration Guide + +## Upgrading to 1.0.0 + +This version removes the option to use "normalization" with clickhouse. It also changes +the schema and database of Airbyte's "raw" tables to be compatible with the new +[Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2) +format. These changes will likely require updates to downstream dbt / SQL models. + +### Database/Schema and the Internal Schema +For other v2 destinations, we have split the raw and final tables into their own schemas, +which in clickhouse is analogous to a `database`. For the Clickhouse destination, this means that +we will only write into the raw table which will live in the `airbyte_internal` database. +The tables written into this schema will be prefixed with either the default database provided in +the `DB Name` field when configuring clickhouse (but can also be overriden in the connection). You can +change the "raw" database from the default `airbyte_internal` by supplying a value for +`Raw Table Schema Name`. + +For Example: + + - DB Name: `default` + - Stream Name: `my_stream` + +Writes to `airbyte_intneral.default_raw__stream_my_stream` + +where as: + + - DB Name: `default` + - Stream Name: `my_stream` + - Raw Table Schema Name: `raw_data` + +Writes to: `raw_data.default_raw__stream_my_stream` diff --git a/docs/integrations/destinations/clickhouse.md b/docs/integrations/destinations/clickhouse.md index 64c3da36f6b8..4495cb79e3da 100644 --- a/docs/integrations/destinations/clickhouse.md +++ b/docs/integrations/destinations/clickhouse.md @@ -89,7 +89,8 @@ Therefore, Airbyte ClickHouse destination will create tables and schemas using t ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------- | +|:--------|:-----------| :--------------------------------------------------------- |:----------------------------------------------------------------------------------------------| +| 1.0.0 | 2024-02-07 | [\#34637](https://github.com/airbytehq/airbyte/pull/34637) | Update the raw table schema | | 0.2.5 | 2023-06-21 | [\#27555](https://github.com/airbytehq/airbyte/pull/27555) | Reduce image size | | 0.2.4 | 2023-06-05 | [\#27036](https://github.com/airbytehq/airbyte/pull/27036) | Internal code change for future development (install normalization packages inside connector) | | 0.2.3 | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing | From 2139544962b9a9e5e621ff3aec70f8eab2f66e32 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 7 Feb 2024 09:56:30 -0800 Subject: [PATCH 63/96] test updates --- .../DestinationAcceptanceTest.java | 27 ++++++------------- .../clickhouse/ClickhouseSqlOperations.java | 2 +- .../ClickhouseDestinationAcceptanceTest.java | 6 +++-- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java index d2aa70a1449c..826d4fb4c0fa 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -790,7 +790,7 @@ public void testIncrementalDedupeSync() throws Exception { .map(record -> Jsons.deserialize(record, AirbyteMessage.class)) .collect(Collectors.toList()); final JsonNode config = getConfig(); - runSyncAndVerifyStateOutput(config, firstSyncMessages, configuredCatalog, true); + runSyncAndVerifyStateOutput(config, firstSyncMessages, configuredCatalog, false); final List secondSyncMessages = Lists.newArrayList( new AirbyteMessage() @@ -821,7 +821,7 @@ public void testIncrementalDedupeSync() throws Exception { .withType(Type.STATE) .withState(new AirbyteStateMessage().withData( Jsons.jsonNode(ImmutableMap.of("checkpoint", 2))))); - runSyncAndVerifyStateOutput(config, secondSyncMessages, configuredCatalog, true); + runSyncAndVerifyStateOutput(config, secondSyncMessages, configuredCatalog, false); final List expectedMessagesAfterSecondSync = new ArrayList<>(); expectedMessagesAfterSecondSync.addAll(firstSyncMessages); @@ -854,22 +854,11 @@ public void testIncrementalDedupeSync() throws Exception { final String defaultSchema = getDefaultSchema(config); retrieveRawRecordsAndAssertSameMessages(catalog, expectedMessagesAfterSecondSync, defaultSchema); - final List actualMessages = retrieveNormalizedRecords(catalog, - defaultSchema); - assertSameMessages(expectedMessages, actualMessages, true); - } - - private String generateBigString(final int addExtraCharacters) { - final int length = getMaxRecordValueLimit() + addExtraCharacters; - return RANDOM - .ints('a', 'z' + 1) - .limit(length) - .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) - .toString(); - } - - protected int getGenerateBigStringAddExtraCharacters() { - return 0; + if (normalizationFromDefinition()) { + final List actualMessages = retrieveNormalizedRecords(catalog, + defaultSchema); + assertSameMessages(expectedMessages, actualMessages, true); + } } /** @@ -1348,7 +1337,7 @@ private List runSync( destination.close(); - if (!runNormalization || (runNormalization && supportsInDestinationNormalization())) { + if (!runNormalization || (supportsInDestinationNormalization())) { return destinationOutput; } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java index 8bfd8442984b..0d0acf62d5ee 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java @@ -37,7 +37,7 @@ public boolean isSchemaRequired() { public String createTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { return String.format( """ - CREATE TABLE IF NOT EXISTS %s.%s ( + CREATE TABLE IF NOT EXISTS `%s`.`%s` ( %s String, %s String, %s DateTime64(3, 'GMT') DEFAULT now(), diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java index 5f5c3ae948fa..0a18db6cd839 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java @@ -21,6 +21,7 @@ import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import java.sql.SQLException; import java.time.Duration; import java.util.HashSet; @@ -111,7 +112,7 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, final String namespace, final JsonNode streamSchema) throws Exception { - return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) + return retrieveRecordsFromTable(StreamId.concatenateRawTableName(namespace, streamName), "airbyte_internal") .stream() .map(r -> Jsons.deserialize(r.get(JavaBaseConstants.COLUMN_NAME_DATA).asText())) .collect(Collectors.toList()); @@ -119,7 +120,8 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { final JdbcDatabase jdbcDB = getDatabase(getConfig()); - final String query = String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT); + final var nameTransformer = new StandardNameTransformer(); + final String query = String.format("SELECT * FROM `%s`.`%s` ORDER BY %s ASC", schemaName, nameTransformer.convertStreamName(tableName), JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT); return jdbcDB.queryJsons(query); } From 541e694395a82a9ec94a1f9be6ca7a59606810dc Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 7 Feb 2024 10:28:51 -0800 Subject: [PATCH 64/96] formatting --- .../clickhouse/ClickhouseDestinationAcceptanceTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java index 0a18db6cd839..c7e7d7a5b6a6 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java @@ -121,7 +121,8 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { final JdbcDatabase jdbcDB = getDatabase(getConfig()); final var nameTransformer = new StandardNameTransformer(); - final String query = String.format("SELECT * FROM `%s`.`%s` ORDER BY %s ASC", schemaName, nameTransformer.convertStreamName(tableName), JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT); + final String query = String.format("SELECT * FROM `%s`.`%s` ORDER BY %s ASC", schemaName, nameTransformer.convertStreamName(tableName), + JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT); return jdbcDB.queryJsons(query); } From 1011b795f80ef5cbddfdb981349b5608cab4ce4f Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 7 Feb 2024 13:31:05 -0800 Subject: [PATCH 65/96] disable ssh tests --- .../clickhouse/SshKeyClickhouseDestinationAcceptanceTest.java | 2 ++ .../SshPasswordClickhouseDestinationAcceptanceTest.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshKeyClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshKeyClickhouseDestinationAcceptanceTest.java index d5ebc6f26f99..4d46da0c56ae 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshKeyClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshKeyClickhouseDestinationAcceptanceTest.java @@ -5,7 +5,9 @@ package io.airbyte.integrations.destination.clickhouse; import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import org.junit.jupiter.api.Disabled; +@Disabled public class SshKeyClickhouseDestinationAcceptanceTest extends SshClickhouseDestinationAcceptanceTest { @Override diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshPasswordClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshPasswordClickhouseDestinationAcceptanceTest.java index 56ccb4d81e1d..27c254f8201b 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshPasswordClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshPasswordClickhouseDestinationAcceptanceTest.java @@ -5,7 +5,9 @@ package io.airbyte.integrations.destination.clickhouse; import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import org.junit.jupiter.api.Disabled; +@Disabled public class SshPasswordClickhouseDestinationAcceptanceTest extends SshClickhouseDestinationAcceptanceTest { @Override From 0c5fcd40ecbfffa3b304b52b56fe28694c572a1b Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Thu, 8 Feb 2024 17:21:23 -0800 Subject: [PATCH 66/96] remove normalization from oracle --- .../integrations/base/JavaBaseConstants.java | 5 ++ ...trictEncryptDestinationAcceptanceTest.java | 6 +- .../destination-oracle/build.gradle | 6 +- .../destination-oracle/metadata.yaml | 16 +++-- .../destination/oracle/OracleDestination.java | 20 +++--- .../oracle/OracleNameTransformer.java | 1 + .../destination/oracle/OracleOperations.java | 61 +++++++++++-------- .../src/main/resources/spec.json | 6 ++ .../SshOracleDestinationAcceptanceTest.java | 4 +- ...ryptedOracleDestinationAcceptanceTest.java | 9 ++- 10 files changed, 86 insertions(+), 48 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/JavaBaseConstants.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/JavaBaseConstants.java index 5001d6119e7a..56c16f137524 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/JavaBaseConstants.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/JavaBaseConstants.java @@ -5,6 +5,7 @@ package io.airbyte.cdk.integrations.base; import java.util.List; +import org.apache.commons.lang3.StringUtils; public final class JavaBaseConstants { @@ -43,4 +44,8 @@ private JavaBaseConstants() {} public static final String DEFAULT_AIRBYTE_INTERNAL_NAMESPACE = "airbyte_internal"; + public static String upperQuoted(final String column) { + return StringUtils.wrap(column.toUpperCase(), "\""); + } + } diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java index 72e2a11ce32f..a37e41918ed1 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java @@ -17,11 +17,11 @@ import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.integrations.destination.oracle.OracleDestination; import io.airbyte.integrations.destination.oracle.OracleNameTransformer; import java.sql.SQLException; import java.util.ArrayList; @@ -73,7 +73,7 @@ protected List retrieveRecords(final TestDestinationEnv env, return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) .stream() .map(r -> Jsons.deserialize( - r.get(OracleDestination.COLUMN_NAME_DATA.replace("\"", "")).asText())) + r.get(JavaBaseConstants.COLUMN_NAME_DATA).asText())) .collect(Collectors.toList()); } @@ -113,7 +113,7 @@ protected List resolveIdentifier(final String identifier) { private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { - final String query = String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, OracleDestination.COLUMN_NAME_EMITTED_AT); + final String query = String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT.toUpperCase()); try (final DSLContext dslContext = getDslContext(config)) { final List result = getDatabase(dslContext).query(ctx -> ctx.fetch(query).stream().toList()); diff --git a/airbyte-integrations/connectors/destination-oracle/build.gradle b/airbyte-integrations/connectors/destination-oracle/build.gradle index a192ee34744a..157740184b44 100644 --- a/airbyte-integrations/connectors/destination-oracle/build.gradle +++ b/airbyte-integrations/connectors/destination-oracle/build.gradle @@ -4,9 +4,9 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' - features = ['db-destinations'] - useLocalCdk = false + cdkVersionRequired = '0.17.0' + features = ['db-destinations', 's3-destinations', 'typing-deduping'] + useLocalCdk = true } //remove once upgrading the CDK version to 0.4.x or later diff --git a/airbyte-integrations/connectors/destination-oracle/metadata.yaml b/airbyte-integrations/connectors/destination-oracle/metadata.yaml index f6f1acf44c37..a4bc8d5420ff 100644 --- a/airbyte-integrations/connectors/destination-oracle/metadata.yaml +++ b/airbyte-integrations/connectors/destination-oracle/metadata.yaml @@ -2,16 +2,12 @@ data: connectorSubtype: database connectorType: destination definitionId: 3986776d-2319-4de9-8af8-db14c0996e72 - dockerImageTag: 0.2.0 + dockerImageTag: 1.0.0 dockerRepository: airbyte/destination-oracle githubIssueLabel: destination-oracle icon: oracle.svg license: ELv2 name: Oracle - normalizationConfig: - normalizationIntegrationType: oracle - normalizationRepository: airbyte/normalization-oracle - normalizationTag: 0.4.3 registries: cloud: dockerRepository: airbyte/destination-oracle-strict-encrypt @@ -21,6 +17,16 @@ data: releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/oracle supportsDbt: true + releases: + breakingChanges: + 1.0.0: + upgradeDeadline: "2024-03-15" + message: > + This version removes the option to use "normalization" with Oracle. It also changes + the schema and database of Airbyte's "raw" tables to be compatible with the new + [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2) + format. These changes will likely require updates to downstream dbt / SQL models. + Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. tags: - language:java ab_internal: diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java index 9a515ef1f74e..d83300f33777 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java @@ -10,9 +10,10 @@ import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.cdk.integrations.base.Destination; import io.airbyte.cdk.integrations.base.IntegrationRunner; -import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination; import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.RawOnlySqlGenerator; import io.airbyte.commons.json.Jsons; import java.io.IOException; import java.io.PrintWriter; @@ -29,13 +30,6 @@ public class OracleDestination extends AbstractJdbcDestination implements Destin private static final Logger LOGGER = LoggerFactory.getLogger(OracleDestination.class); public static final String DRIVER_CLASS = DatabaseDriver.ORACLE.getDriverClassName(); - public static final String COLUMN_NAME_AB_ID = - "\"" + JavaBaseConstants.COLUMN_NAME_AB_ID.toUpperCase() + "\""; - public static final String COLUMN_NAME_DATA = - "\"" + JavaBaseConstants.COLUMN_NAME_DATA.toUpperCase() + "\""; - public static final String COLUMN_NAME_EMITTED_AT = - "\"" + JavaBaseConstants.COLUMN_NAME_EMITTED_AT.toUpperCase() + "\""; - protected static final String KEY_STORE_FILE_PATH = "clientkeystore.jks"; private static final String KEY_STORE_PASS = RandomStringUtils.randomAlphanumeric(8); public static final String ENCRYPTION_METHOD_KEY = "encryption_method"; @@ -134,6 +128,16 @@ private static void tryConvertAndImportCertificate(final String certificate) { } } + @Override + public boolean isV2Destination() { + return true; + } + + @Override + protected JdbcSqlGenerator getSqlGenerator() { + return new RawOnlySqlGenerator(new OracleNameTransformer()); + } + private static void convertAndImportCertificate(final String certificate) throws IOException, InterruptedException { final Runtime run = Runtime.getRuntime(); diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleNameTransformer.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleNameTransformer.java index ace575355050..18fbe6995071 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleNameTransformer.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleNameTransformer.java @@ -17,6 +17,7 @@ public String applyDefaultCase(final String input) { } @Override + @Deprecated public String getRawTableName(final String streamName) { return convertStreamName("airbyte_raw_" + streamName); } diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java index 468e33bd7345..dd55caed976e 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java @@ -4,15 +4,19 @@ package io.airbyte.integrations.destination.oracle; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.upperQuoted; + import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; import io.airbyte.commons.json.Jsons; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Timestamp; +import java.sql.Types; import java.time.Instant; import java.util.List; import java.util.UUID; @@ -58,18 +62,24 @@ public void createTableIfNotExists(final JdbcDatabase database, final String sch @Override public String createTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { return String.format( - "CREATE TABLE %s.%s ( \n" - + "%s VARCHAR(64) PRIMARY KEY,\n" - + "%s NCLOB,\n" - + "%s TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP\n" - + ")", - schemaName, tableName, - OracleDestination.COLUMN_NAME_AB_ID, OracleDestination.COLUMN_NAME_DATA, OracleDestination.COLUMN_NAME_EMITTED_AT, - OracleDestination.COLUMN_NAME_DATA); + """ + CREATE TABLE %s.%s ( + %s VARCHAR(64) PRIMARY KEY, + %s NCLOB, + %s TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP + %s TIMESTAMP WITH TIME ZONE DEFAULT NULL + ) + """, + schemaName, tableName, + upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID), + upperQuoted(JavaBaseConstants.COLUMN_NAME_DATA), + upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT), + upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT) + ); } private boolean tableExists(final JdbcDatabase database, final String schemaName, final String tableName) throws Exception { - final Integer count = database.queryInt("select count(*) \n from all_tables\n where upper(owner) = upper(?) and upper(table_name) = upper(?)", + final int count = database.queryInt("select count(*) \n from all_tables\n where upper(owner) = upper(?) and upper(table_name) = upper(?)", schemaName, tableName); return count == 1; } @@ -94,23 +104,25 @@ public String truncateTableQuery(final JdbcDatabase database, final String schem @Override public void insertRecords(final JdbcDatabase database, - final List records, + final List records, final String schemaName, final String tempTableName) throws Exception { final String tableName = String.format("%s.%s", schemaName, tempTableName); - final String columns = String.format("(%s, %s, %s)", - OracleDestination.COLUMN_NAME_AB_ID, OracleDestination.COLUMN_NAME_DATA, OracleDestination.COLUMN_NAME_EMITTED_AT); - final String recordQueryComponent = "(?, ?, ?)\n"; - insertRawRecordsInSingleQuery(tableName, columns, recordQueryComponent, database, records, UUID::randomUUID); + final String columns = String.format("(%s, %s, %s, %s)", + upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID), + upperQuoted(JavaBaseConstants.COLUMN_NAME_DATA), + upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT), + upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT) + ); + insertRawRecordsInSingleQuery(tableName, columns, database, records, UUID::randomUUID); } // Adapted from SqlUtils.insertRawRecordsInSingleQuery to meet some needs specific to Oracle syntax private static void insertRawRecordsInSingleQuery(final String tableName, final String columns, - final String recordQueryComponent, final JdbcDatabase jdbcDatabase, - final List records, + final List records, final Supplier uuidSupplier) throws SQLException { if (records.isEmpty()) { @@ -129,20 +141,20 @@ private static void insertRawRecordsInSingleQuery(final String tableName, // The "SELECT 1 FROM DUAL" at the end is a formality to satisfy the needs of the Oracle syntax. // (see https://stackoverflow.com/a/93724 for details) final StringBuilder sql = new StringBuilder("INSERT ALL "); - records.forEach(r -> sql.append(String.format("INTO %s %s VALUES %s", tableName, columns, recordQueryComponent))); + records.forEach(r -> sql.append(String.format("INTO %s %s VALUES %s", tableName, columns, "(?, ?, ?, ?)\n"))); sql.append(" SELECT 1 FROM DUAL"); final String query = sql.toString(); try (final PreparedStatement statement = connection.prepareStatement(query)) { // second loop: bind values to the SQL string. int i = 1; - for (final AirbyteRecordMessage message : records) { + for (final PartialAirbyteMessage message : records) { // 1-indexed - final JsonNode formattedData = StandardNameTransformer.formatJsonPath(message.getData()); - statement.setString(i, uuidSupplier.get().toString()); - statement.setString(i + 1, Jsons.serialize(formattedData)); - statement.setTimestamp(i + 2, Timestamp.from(Instant.ofEpochMilli(message.getEmittedAt()))); - i += 3; + final JsonNode formattedData = StandardNameTransformer.formatJsonPath(message.getRecord().getData()); + statement.setString(i++, uuidSupplier.get().toString()); + statement.setString(i++, Jsons.serialize(formattedData)); + statement.setTimestamp(i++, Timestamp.from(Instant.ofEpochMilli(message.getRecord().getEmittedAt()))); + statement.setNull(i++, Types.TIMESTAMP); } statement.execute(); @@ -173,5 +185,4 @@ public boolean isValidData(final JsonNode data) { public boolean isSchemaRequired() { return true; } - } diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-oracle/src/main/resources/spec.json index 35aa4090b786..c5cd4f20adf2 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-oracle/src/main/resources/spec.json @@ -120,6 +120,12 @@ } } ] + }, + "raw_data_schema": { + "type": "string", + "description": "The schema to write raw tables into (default: airbyte_internal)", + "title": "Raw Table Schema Name", + "order": 7 } } } diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java index 10ce0fde6c7a..aa54a3348ed5 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java @@ -4,6 +4,8 @@ package io.airbyte.integrations.destination.oracle; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.upperQuoted; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; @@ -124,7 +126,7 @@ private List retrieveRecordsFromTable(final String tableName, final St (CheckedFunction, Exception>) mangledConfig -> getDatabaseFromConfig(mangledConfig) .query( ctx -> ctx - .fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, OracleDestination.COLUMN_NAME_EMITTED_AT))) + .fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT)))) .stream() .map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat())) .map(Jsons::deserialize) diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java index d87d36041168..28eef3951b89 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.destination.oracle; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.upperQuoted; import static io.airbyte.cdk.integrations.util.HostPortResolver.resolveHost; import static io.airbyte.cdk.integrations.util.HostPortResolver.resolvePort; import static org.hamcrest.CoreMatchers.equalTo; @@ -20,11 +21,13 @@ import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import java.sql.SQLException; import java.util.ArrayList; import java.util.HashSet; @@ -72,10 +75,10 @@ protected List retrieveRecords(final TestDestinationEnv env, final String namespace, final JsonNode streamSchema) throws Exception { - return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) + return retrieveRecordsFromTable(namingResolver.convertStreamName(StreamId.concatenateRawTableName(namespace, streamName)), namespace) .stream() .map(r -> Jsons.deserialize( - r.get(OracleDestination.COLUMN_NAME_DATA.replace("\"", "")).asText())) + r.get(JavaBaseConstants.COLUMN_NAME_DATA.toUpperCase()).asText())) .collect(Collectors.toList()); } @@ -126,7 +129,7 @@ private List retrieveRecordsFromTable(final String tableName, final St final List result = getDatabase(dslContext) .query(ctx -> new ArrayList<>(ctx.fetch( String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, - OracleDestination.COLUMN_NAME_EMITTED_AT)))); + upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT))))); return result .stream() .map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat())) From c486e3e10f24b050e1e3a214c7ece0372a42ff67 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Tue, 13 Feb 2024 17:18:27 -0800 Subject: [PATCH 67/96] java 21 upgrade --- airbyte-cdk/java/airbyte-cdk/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/build.gradle b/airbyte-cdk/java/airbyte-cdk/build.gradle index 7696b46dd69a..4b3132396c05 100644 --- a/airbyte-cdk/java/airbyte-cdk/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/build.gradle @@ -51,13 +51,13 @@ subprojects { subproject -> compileKotlin { compilerOptions { - jvmTarget = JvmTarget.JVM_17 + jvmTarget = JvmTarget.JVM_21 languageVersion = KotlinVersion.KOTLIN_1_9 } } compileTestKotlin { compilerOptions { - jvmTarget = JvmTarget.JVM_17 + jvmTarget = JvmTarget.JVM_21 languageVersion = KotlinVersion.KOTLIN_1_9 } } From 59219c60ddc8c4f78a1a1b99036a147a4d4ffaa6 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 14 Feb 2024 14:06:29 -0800 Subject: [PATCH 68/96] java 21 upgrade --- .../bases/base-normalization/build.gradle | 6 ++++-- .../build.gradle | 6 +++--- .../metadata.yaml | 16 +++++++++++----- .../destination-clickhouse/build.gradle | 2 +- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle index 0be8b3f954d4..13f2dd53c9f9 100644 --- a/airbyte-integrations/bases/base-normalization/build.gradle +++ b/airbyte-integrations/bases/base-normalization/build.gradle @@ -1,5 +1,3 @@ -import org.apache.tools.ant.taskdefs.condition.Os - plugins { id 'airbyte-docker-legacy' id 'airbyte-python' @@ -38,6 +36,10 @@ tasks.named('check').configure { dependsOn generate } +tasks.named("jar").configure { + dependsOn copySshScript +} + [ 'bigquery', 'mysql', diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle index d1a316d740a4..a799c3b2598b 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle @@ -4,9 +4,9 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' - features = ['db-destinations'] - useLocalCdk = false + cdkVersionRequired = '0.21.0' + features = ['db-destinations', 's3-destinations', 'typing-deduping'] + useLocalCdk = true } //remove once upgrading the CDK version to 0.4.x or later diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/metadata.yaml index b58fc5f5d3e5..c5023258510c 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/metadata.yaml @@ -7,16 +7,22 @@ data: connectorSubtype: database connectorType: destination definitionId: ce0d828e-1dc4-496c-b122-2da42e637e48 - dockerImageTag: 0.2.5 + dockerImageTag: 1.0.0 dockerRepository: airbyte/destination-clickhouse-strict-encrypt githubIssueLabel: destination-clickhouse icon: clickhouse.svg license: MIT name: Clickhouse - normalizationConfig: - normalizationIntegrationType: clickhouse - normalizationRepository: airbyte/normalization-clickhouse - normalizationTag: 0.4.1 + releases: + breakingChanges: + 1.0.0: + upgradeDeadline: "2024-03-15" + message: > + This version removes the option to use "normalization" with clickhouse. It also changes + the schema and database of Airbyte's "raw" tables to be compatible with the new + [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2) + format. These changes will likely require updates to downstream dbt / SQL models. + Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/clickhouse supportsDbt: false diff --git a/airbyte-integrations/connectors/destination-clickhouse/build.gradle b/airbyte-integrations/connectors/destination-clickhouse/build.gradle index daf34f9f933a..ec3e75139b09 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.17.0' + cdkVersionRequired = '0.21.0' features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = true } From 40f934793bc93edba0dce1795051e51a5b01a37f Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 14 Feb 2024 14:19:05 -0800 Subject: [PATCH 69/96] disable tests comment --- .../clickhouse/SshKeyClickhouseDestinationAcceptanceTest.java | 2 +- .../SshPasswordClickhouseDestinationAcceptanceTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshKeyClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshKeyClickhouseDestinationAcceptanceTest.java index 4d46da0c56ae..b3863d321e62 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshKeyClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshKeyClickhouseDestinationAcceptanceTest.java @@ -7,7 +7,7 @@ import io.airbyte.cdk.integrations.base.ssh.SshTunnel; import org.junit.jupiter.api.Disabled; -@Disabled +@Disabled("This test was failing on the master branch and seems unrelated to any changes in the PR") public class SshKeyClickhouseDestinationAcceptanceTest extends SshClickhouseDestinationAcceptanceTest { @Override diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshPasswordClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshPasswordClickhouseDestinationAcceptanceTest.java index 27c254f8201b..aa66acd75ec5 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshPasswordClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshPasswordClickhouseDestinationAcceptanceTest.java @@ -7,7 +7,7 @@ import io.airbyte.cdk.integrations.base.ssh.SshTunnel; import org.junit.jupiter.api.Disabled; -@Disabled +@Disabled("This test was failing on the master branch and seems unrelated to any changes in the PR") public class SshPasswordClickhouseDestinationAcceptanceTest extends SshClickhouseDestinationAcceptanceTest { @Override From 4908a81bc1b61521755e096878c82776ff8cef85 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 14 Feb 2024 14:58:39 -0800 Subject: [PATCH 70/96] pr feedback --- .../airbyte/cdk/integrations/util/ConfiguredCatalogUtil.kt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConfiguredCatalogUtil.kt b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConfiguredCatalogUtil.kt index 44d8245effd6..42183f51fcbe 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConfiguredCatalogUtil.kt +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConfiguredCatalogUtil.kt @@ -1,7 +1,6 @@ package io.airbyte.cdk.integrations.util import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog -import org.apache.commons.lang3.StringUtils /** * For streams in [catalog] which do not have a namespace specified, explicitly set their namespace @@ -13,9 +12,9 @@ import org.apache.commons.lang3.StringUtils } // TODO: This logic exists in all V2 destinations. // This is sad that if we forget to add this, there will be a null pointer during parseCatalog - for (stream in catalog.streams) { - if (StringUtils.isEmpty(stream.stream.namespace)) { - stream.stream.namespace = defaultNamespace + for (catalogStream in catalog.streams) { + if (catalogStream.stream.namespace.isNullOrEmpty()) { + catalogStream.stream.namespace = defaultNamespace } } } From 32fcb0ffe4afded85e1358cff9c5370a7c1e608a Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 14 Feb 2024 16:59:31 -0800 Subject: [PATCH 71/96] pr feedback --- .../destination/DestinationAcceptanceTest.java | 6 +++++- .../clickhouse/SshClickhouseDestinationAcceptanceTest.java | 7 ++++--- .../SshKeyClickhouseDestinationAcceptanceTest.java | 2 -- .../SshPasswordClickhouseDestinationAcceptanceTest.java | 2 -- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java index 5cf03352e8e5..8094c8fc214c 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -789,7 +789,7 @@ public void testIncrementalDedupeSync() throws Exception { .map(record -> Jsons.deserialize(record, AirbyteMessage.class)) .collect(Collectors.toList()); final JsonNode config = getConfig(); - runSyncAndVerifyStateOutput(config, firstSyncMessages, configuredCatalog, false); + runSyncAndVerifyStateOutput(config, firstSyncMessages, configuredCatalog, supportsNormalization()); final List secondSyncMessages = Lists.newArrayList( new AirbyteMessage() @@ -1849,6 +1849,10 @@ public Stream provideArguments(final ExtensionContext conte } + private boolean supportsNormalization() { + return supportsInDestinationNormalization() || normalizationFromDefinition(); + } + private static V0 convertProtocolObject(final V1 v1, final Class klass) { return Jsons.object(Jsons.jsonNode(v1), klass); } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java index c82dfca207c1..5d3c45f82240 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java @@ -19,6 +19,7 @@ import io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider; import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -85,7 +86,7 @@ protected String getDefaultSchema(final JsonNode config) { @Override protected JsonNode getConfig() throws Exception { return bastion.getTunnelConfig(getTunnelMethod(), bastion.getBasicDbConfigBuider(db, DB_NAME) - .put("schema", DB_NAME), false); + .put("schema", DB_NAME), true); } @Override @@ -109,7 +110,7 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, final String namespace, final JsonNode streamSchema) throws Exception { - return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) + return retrieveRecordsFromTable(StreamId.concatenateRawTableName(namespace, streamName), "airbyte_internal") .stream() .map(r -> Jsons.deserialize(r.get(JavaBaseConstants.COLUMN_NAME_DATA).asText())) .collect(Collectors.toList()); @@ -122,7 +123,7 @@ private List retrieveRecordsFromTable(final String tableName, final St JdbcUtils.PORT_LIST_KEY, mangledConfig -> { final JdbcDatabase database = getDatabase(mangledConfig); - final String query = String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT); + final String query = String.format("SELECT * FROM `%s`.`%s` ORDER BY %s ASC", schemaName, namingResolver.convertStreamName(tableName), JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT); return database.queryJsons(query); }); } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshKeyClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshKeyClickhouseDestinationAcceptanceTest.java index b3863d321e62..d5ebc6f26f99 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshKeyClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshKeyClickhouseDestinationAcceptanceTest.java @@ -5,9 +5,7 @@ package io.airbyte.integrations.destination.clickhouse; import io.airbyte.cdk.integrations.base.ssh.SshTunnel; -import org.junit.jupiter.api.Disabled; -@Disabled("This test was failing on the master branch and seems unrelated to any changes in the PR") public class SshKeyClickhouseDestinationAcceptanceTest extends SshClickhouseDestinationAcceptanceTest { @Override diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshPasswordClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshPasswordClickhouseDestinationAcceptanceTest.java index aa66acd75ec5..56ccb4d81e1d 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshPasswordClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshPasswordClickhouseDestinationAcceptanceTest.java @@ -5,9 +5,7 @@ package io.airbyte.integrations.destination.clickhouse; import io.airbyte.cdk.integrations.base.ssh.SshTunnel; -import org.junit.jupiter.api.Disabled; -@Disabled("This test was failing on the master branch and seems unrelated to any changes in the PR") public class SshPasswordClickhouseDestinationAcceptanceTest extends SshClickhouseDestinationAcceptanceTest { @Override From da317762928939b5dc992fdf23a195df33a785a3 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 14 Feb 2024 17:11:24 -0800 Subject: [PATCH 72/96] formatting --- .../clickhouse/SshClickhouseDestinationAcceptanceTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java index 5d3c45f82240..163c9a6e36c6 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java @@ -123,7 +123,8 @@ private List retrieveRecordsFromTable(final String tableName, final St JdbcUtils.PORT_LIST_KEY, mangledConfig -> { final JdbcDatabase database = getDatabase(mangledConfig); - final String query = String.format("SELECT * FROM `%s`.`%s` ORDER BY %s ASC", schemaName, namingResolver.convertStreamName(tableName), JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT); + final String query = String.format("SELECT * FROM `%s`.`%s` ORDER BY %s ASC", schemaName, namingResolver.convertStreamName(tableName), + JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT); return database.queryJsons(query); }); } From 19b6ab4dd550d3d88f4eacff5da9e62f04b04b8a Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Thu, 15 Feb 2024 18:01:59 -0800 Subject: [PATCH 73/96] fix strict encrypt tests --- .../destination/StandardNameTransformer.java | 2 +- .../ClickhouseDestinationStrictEncrypt.java | 4 ++++ ...ckhouseDestinationStrictEncryptAcceptanceTest.java | 7 +++++-- .../destinations/clickhouse-migrations.md | 11 ++++++++--- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StandardNameTransformer.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StandardNameTransformer.java index 8601f70c49e6..c92345a22b8f 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StandardNameTransformer.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StandardNameTransformer.java @@ -31,7 +31,7 @@ public String getNamespace(final String namespace) { } @Override - @Deprecated +// @Deprecated see https://github.com/airbytehq/airbyte/issues/35333 public String getRawTableName(final String streamName) { return convertStreamName("_airbyte_raw_" + streamName); } diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncrypt.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncrypt.java index 98d998a9e3ef..17c49aa3f8d3 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncrypt.java +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncrypt.java @@ -36,4 +36,8 @@ public static void main(final String[] args) throws Exception { LOGGER.info("completed destination: {}", ClickhouseDestinationStrictEncrypt.class); } + @Override + public boolean isV2Destination() { + return true; + } } diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java index 6769060d4ff1..b999c62a8776 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java @@ -21,6 +21,7 @@ import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import java.sql.SQLException; import java.time.Duration; import java.util.ArrayList; @@ -139,7 +140,7 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, final String namespace, final JsonNode streamSchema) throws Exception { - return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) + return retrieveRecordsFromTable(StreamId.concatenateRawTableName(namespace, streamName), "airbyte_internal") .stream() .map(r -> Jsons.deserialize(r.get(JavaBaseConstants.COLUMN_NAME_DATA).asText())) .collect(Collectors.toList()); @@ -147,7 +148,9 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { final JdbcDatabase jdbcDB = getDatabase(getConfig()); - final String query = String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT); + final var nameTransformer = new StandardNameTransformer(); + final String query = String.format("SELECT * FROM `%s`.`%s` ORDER BY %s ASC", schemaName, nameTransformer.convertStreamName(tableName), + JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT); return jdbcDB.queryJsons(query); } diff --git a/docs/integrations/destinations/clickhouse-migrations.md b/docs/integrations/destinations/clickhouse-migrations.md index 8386e911b804..5f1a715d40c1 100644 --- a/docs/integrations/destinations/clickhouse-migrations.md +++ b/docs/integrations/destinations/clickhouse-migrations.md @@ -5,14 +5,19 @@ This version removes the option to use "normalization" with clickhouse. It also changes the schema and database of Airbyte's "raw" tables to be compatible with the new [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2) -format. These changes will likely require updates to downstream dbt / SQL models. +format. These changes will likely require updates to downstream dbt / SQL models. After this update, +Airbyte will only produce the ‘raw’ v2 tables, which store all content in JSON. These changes remove +the ability to do deduplicated syncs with Clickhouse. (Clickhouse has an overview)[[https://clickhouse.com/docs/en/integrations/dbt]] +for integrating with dbt If you are interested in the Clickhouse destination gaining the full features +of Destinations V2 (including final tables), click [[https://github.com/airbytehq/airbyte/discussions/35339]] +to register your interest. ### Database/Schema and the Internal Schema -For other v2 destinations, we have split the raw and final tables into their own schemas, +We have split the raw and final tables into their own schemas, which in clickhouse is analogous to a `database`. For the Clickhouse destination, this means that we will only write into the raw table which will live in the `airbyte_internal` database. The tables written into this schema will be prefixed with either the default database provided in -the `DB Name` field when configuring clickhouse (but can also be overriden in the connection). You can +the `DB Name` field when configuring clickhouse (but can also be overridden in the connection). You can change the "raw" database from the default `airbyte_internal` by supplying a value for `Raw Table Schema Name`. From 369ffdc5c643fb4df983919ba5582f9ea683e7bb Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Thu, 15 Feb 2024 18:07:05 -0800 Subject: [PATCH 74/96] formatting --- .../cdk/integrations/destination/StandardNameTransformer.java | 2 +- .../clickhouse/ClickhouseDestinationStrictEncrypt.java | 1 + .../ClickhouseDestinationStrictEncryptAcceptanceTest.java | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StandardNameTransformer.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StandardNameTransformer.java index c92345a22b8f..cc9c2dc4cd15 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StandardNameTransformer.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StandardNameTransformer.java @@ -31,7 +31,7 @@ public String getNamespace(final String namespace) { } @Override -// @Deprecated see https://github.com/airbytehq/airbyte/issues/35333 + // @Deprecated see https://github.com/airbytehq/airbyte/issues/35333 public String getRawTableName(final String streamName) { return convertStreamName("_airbyte_raw_" + streamName); } diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncrypt.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncrypt.java index 17c49aa3f8d3..4efc4db3545c 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncrypt.java +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncrypt.java @@ -40,4 +40,5 @@ public static void main(final String[] args) throws Exception { public boolean isV2Destination() { return true; } + } diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java index b999c62a8776..991ef0e2cde4 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java @@ -150,7 +150,7 @@ private List retrieveRecordsFromTable(final String tableName, final St final JdbcDatabase jdbcDB = getDatabase(getConfig()); final var nameTransformer = new StandardNameTransformer(); final String query = String.format("SELECT * FROM `%s`.`%s` ORDER BY %s ASC", schemaName, nameTransformer.convertStreamName(tableName), - JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT); + JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT); return jdbcDB.queryJsons(query); } From 9ab52ae9b13b8702bc6290fa3c1cb65be39fe0b0 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Fri, 16 Feb 2024 15:38:56 -0800 Subject: [PATCH 75/96] clickhouse migration query --- .../destinations/clickhouse-migrations.md | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/integrations/destinations/clickhouse-migrations.md b/docs/integrations/destinations/clickhouse-migrations.md index 5f1a715d40c1..df8590b36a56 100644 --- a/docs/integrations/destinations/clickhouse-migrations.md +++ b/docs/integrations/destinations/clickhouse-migrations.md @@ -12,6 +12,35 @@ for integrating with dbt If you are interested in the Clickhouse destination gai of Destinations V2 (including final tables), click [[https://github.com/airbytehq/airbyte/discussions/35339]] to register your interest. +This upgrade will ignore any existing raw tables and will not migrate any data to the new schema. +For each stream, you could perform the following query to migrate the data from the old raw table +to the new raw table: + +```sql +-- assumes your database was 'default' +-- replace `{{stream_name}}` with replace your stream name + +CREATE TABLE airbyte_internal.default_raw__stream_{{stream_name}} +( + `_airbyte_raw_id` String, + `_airbyte_extracted_at` DateTime64(3, 'GMT') DEFAULT now(), + `_airbyte_loaded_at` DateTime64(3, 'GMT') NULL, + `_airbyte_data` String, + PRIMARY KEY(`_airbyte_raw_id`) +) +ENGINE = MergeTree; + +INSERT INTO `airbyte_internal`.`default_raw__stream_{{stream_name}}` + SELECT + `_airbyte_ab_id` AS "_airbyte_raw_id", + `_airbyte_emitted_at` AS "_airbyte_extracted_at", + NULL AS "_airbyte_loaded_at", + _airbyte_data AS "_airbyte_data" + FROM default._airbyte_raw_{{stream_name}}; +``` + +Airbyte will not delete any of your v1 data. + ### Database/Schema and the Internal Schema We have split the raw and final tables into their own schemas, which in clickhouse is analogous to a `database`. For the Clickhouse destination, this means that From 890ef954b4df1bd5a64e5c82eb41de0dffd0fd5a Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Tue, 20 Feb 2024 16:07:36 -0800 Subject: [PATCH 76/96] remove append option --- .../destination-clickhouse/src/main/resources/spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json index 26b6fb701381..a86a89f7f746 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json @@ -3,7 +3,7 @@ "supportsIncremental": true, "supportsNormalization": true, "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"], + "supported_destination_sync_modes": ["overwrite", "append"], "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "ClickHouse Destination Spec", From 907eb1cd8a713cf58cb88837a35d49f7d89d9434 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 21 Feb 2024 09:57:20 -0800 Subject: [PATCH 77/96] update tests and docs --- .../build.gradle | 6 +- .../metadata.yaml | 14 ++-- .../destination-oracle/build.gradle | 2 +- .../NneOracleDestinationAcceptanceTest.java | 11 +++- .../destination/oracle/OracleContainer.java | 2 +- ...ryptedOracleDestinationAcceptanceTest.java | 25 ++++--- .../destinations/oracle-migrations.md | 65 +++++++++++++++++++ docs/integrations/destinations/oracle.md | 3 +- 8 files changed, 101 insertions(+), 27 deletions(-) create mode 100644 docs/integrations/destinations/oracle-migrations.md diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle index 0e940345ab00..3b2ffa30ce59 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle @@ -4,9 +4,9 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' - features = ['db-destinations'] - useLocalCdk = false + cdkVersionRequired = '0.21.0' + features = ['db-destinations', 's3-destinations', 'typing-deduping'] + useLocalCdk = true } //remove once upgrading the CDK version to 0.4.x or later diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/metadata.yaml index bd323c6fdbc5..2b4dbc001b7a 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/metadata.yaml @@ -13,11 +13,17 @@ data: icon: oracle.svg license: ELv2 name: Oracle - normalizationConfig: - normalizationIntegrationType: oracle - normalizationRepository: airbyte/normalization-oracle - normalizationTag: 0.4.1 releaseStage: alpha + releases: + breakingChanges: + 1.0.0: + upgradeDeadline: "2024-03-15" + message: > + This version removes the option to use "normalization" with Oracle. It also changes + the schema and database of Airbyte's "raw" tables to be compatible with the new + [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2) + format. These changes will likely require updates to downstream dbt / SQL models. + Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. documentationUrl: https://docs.airbyte.com/integrations/destinations/oracle supportsDbt: true tags: diff --git a/airbyte-integrations/connectors/destination-oracle/build.gradle b/airbyte-integrations/connectors/destination-oracle/build.gradle index 157740184b44..49eb4a08ad79 100644 --- a/airbyte-integrations/connectors/destination-oracle/build.gradle +++ b/airbyte-integrations/connectors/destination-oracle/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.17.0' + cdkVersionRequired = '0.21.0' features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = true } diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java index fc946fdddae9..0a2d2daff0d9 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java @@ -18,6 +18,7 @@ import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.commons.json.Jsons; import java.sql.SQLException; +import java.time.Duration; import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; @@ -42,8 +43,12 @@ public void testEncryption() throws SQLException { String.format(DatabaseDriver.ORACLE.getUrlFormatString(), config.get("host").asText(), config.get("port").asInt(), - config.get("sid").asText()), - getAdditionalProperties(algorithm))); + config.get("sid").asText() + ), + getAdditionalProperties(algorithm), + Duration.ofMinutes(5) + ) + ); final String networkServiceBanner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; @@ -78,7 +83,7 @@ public void testCheckProtocol() throws SQLException { clone.get("host").asText(), clone.get("port").asInt(), clone.get("sid").asText()), - getAdditionalProperties(algorithm))); + getAdditionalProperties(algorithm), Duration.ofMinutes(5))); final String networkServiceBanner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual"; final List collect = database.queryJsons(networkServiceBanner); diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/OracleContainer.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/OracleContainer.java index eaee5d0ea23c..d7e99af18564 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/OracleContainer.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/OracleContainer.java @@ -22,7 +22,7 @@ public class OracleContainer extends JdbcDatabaseContainer { public static final String NAME = "oracle"; private static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("gvenzl/oracle-xe"); - static final String DEFAULT_TAG = "18.4.0-slim"; + static final String DEFAULT_TAG = "21.3.0-slim"; static final String IMAGE = DEFAULT_IMAGE_NAME.getUnversionedPart(); private static final int ORACLE_PORT = 1521; diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java index 28eef3951b89..c86eff950713 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java @@ -125,17 +125,16 @@ protected JsonNode getFailCheckConfig() { private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { - try (final DSLContext dslContext = getDSLContext(config)) { - final List result = getDatabase(dslContext) - .query(ctx -> new ArrayList<>(ctx.fetch( - String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, - upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT))))); - return result - .stream() - .map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat())) - .map(Jsons::deserialize) - .collect(Collectors.toList()); - } + final DSLContext dslContext = getDSLContext(config); + final List result = getDatabase(dslContext) + .query(ctx -> new ArrayList<>(ctx.fetch( + String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, + upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT))))); + return result + .stream() + .map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat())) + .map(Jsons::deserialize) + .collect(Collectors.toList()); } private static DSLContext getDSLContext(final JsonNode config) { @@ -162,15 +161,13 @@ protected void setup(final TestDestinationEnv testEnv, HashSet TEST_SCHE db.start(); config = getConfig(db); - - try (final DSLContext dslContext = getDSLContext(config)) { + final DSLContext dslContext = getDSLContext(config); final Database database = getDatabase(dslContext); database.query( ctx -> ctx.fetch(String.format("CREATE USER %s IDENTIFIED BY %s", schemaName, schemaName))); database.query(ctx -> ctx.fetch(String.format("GRANT ALL PRIVILEGES TO %s", schemaName))); ((ObjectNode) config).put(JdbcUtils.SCHEMA_KEY, dbName); - } } @Override diff --git a/docs/integrations/destinations/oracle-migrations.md b/docs/integrations/destinations/oracle-migrations.md new file mode 100644 index 000000000000..478694083737 --- /dev/null +++ b/docs/integrations/destinations/oracle-migrations.md @@ -0,0 +1,65 @@ +# Clickhouse Migration Guide + +## Upgrading to 1.0.0 + +This version removes the option to use "normalization" with clickhouse. It also changes +the schema and database of Airbyte's "raw" tables to be compatible with the new +[Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2) +format. These changes will likely require updates to downstream dbt / SQL models. After this update, +Airbyte will only produce the ‘raw’ v2 tables, which store all content in JSON. These changes remove +the ability to do deduplicated syncs with Clickhouse. (Clickhouse has an overview)[[https://clickhouse.com/docs/en/integrations/dbt]] +for integrating with dbt If you are interested in the Clickhouse destination gaining the full features +of Destinations V2 (including final tables), click [[https://github.com/airbytehq/airbyte/discussions/35339]] +to register your interest. + +This upgrade will ignore any existing raw tables and will not migrate any data to the new schema. +For each stream, you could perform the following query to migrate the data from the old raw table +to the new raw table: + +```sql +-- assumes your database was 'default' +-- replace `{{stream_name}}` with replace your stream name + +CREATE TABLE airbyte_internal.default_raw__stream_{{stream_name}} +( + `_airbyte_raw_id` String, + `_airbyte_extracted_at` DateTime64(3, 'GMT') DEFAULT now(), + `_airbyte_loaded_at` DateTime64(3, 'GMT') NULL, + `_airbyte_data` String, + PRIMARY KEY(`_airbyte_raw_id`) +) +ENGINE = MergeTree; + +INSERT INTO `airbyte_internal`.`default_raw__stream_{{stream_name}}` + SELECT + `_airbyte_ab_id` AS "_airbyte_raw_id", + `_airbyte_emitted_at` AS "_airbyte_extracted_at", + NULL AS "_airbyte_loaded_at", + _airbyte_data AS "_airbyte_data" + FROM default._airbyte_raw_{{stream_name}}; +``` + +Airbyte will not delete any of your v1 data. + +### Database/Schema and the Internal Schema +We have split the raw and final tables into their own schemas, which means that +we will only write into the raw table which will live in the `airbyte_internal` schema. +The tables written into this schema will be prefixed with either the default database provided in +the `DB Name` field when configuring clickhouse (but can also be overridden in the connection). You can +change the "raw" database from the default `airbyte_internal` by supplying a value for +`Raw Table Schema Name`. + +For Example: + +- Schema: `default` +- Stream Name: `my_stream` + +Writes to `airbyte_intneral.default_raw__stream_my_stream` + +where as: + +- DB Name: `default` +- Stream Name: `my_stream` +- Raw Table Schema Name: `raw_data` + +Writes to: `raw_data.default_raw__stream_my_stream` diff --git a/docs/integrations/destinations/oracle.md b/docs/integrations/destinations/oracle.md index d2e9867eb04a..fe0570a6f15f 100644 --- a/docs/integrations/destinations/oracle.md +++ b/docs/integrations/destinations/oracle.md @@ -91,7 +91,8 @@ Airbyte has the ability to connect to the Oracle source with 3 network connectiv ## Changelog | Version | Date | Pull Request | Subject | -| :---------- | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------------- | +|:------------|:-----------| :--------------------------------------------------------- |:----------------------------------------------------------------------------------------------------| +| 1.0.0 | 2024-02-20 | [\#35470](https://github.com/airbytehq/airbyte/pull/35470) | Removes Normalization, updates to V2 Raw Table Format | | 0.2.0 | 2023-06-27 | [\#27781](https://github.com/airbytehq/airbyte/pull/27781) | License Update: Elv2 | | 0.1.19 | 2022-07-26 | [\#10719](https://github.com/airbytehq/airbyte/pull/) | Destination Oracle: added custom JDBC parameters support. | | 0.1.18 | 2022-07-14 | [\#14618](https://github.com/airbytehq/airbyte/pull/14618) | Removed additionalProperties: false from JDBC destination connectors | From 89ebbc75a8d63327b9152a16be4cfa21915c4e1d Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 21 Feb 2024 10:31:02 -0800 Subject: [PATCH 78/96] use non local cdk version --- .../destination-clickhouse-strict-encrypt/build.gradle | 2 +- .../connectors/destination-clickhouse/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle index a799c3b2598b..abb7773e3bb6 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle @@ -6,7 +6,7 @@ plugins { airbyteJavaConnector { cdkVersionRequired = '0.21.0' features = ['db-destinations', 's3-destinations', 'typing-deduping'] - useLocalCdk = true + useLocalCdk = false } //remove once upgrading the CDK version to 0.4.x or later diff --git a/airbyte-integrations/connectors/destination-clickhouse/build.gradle b/airbyte-integrations/connectors/destination-clickhouse/build.gradle index ec3e75139b09..3b921450798b 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse/build.gradle @@ -6,7 +6,7 @@ plugins { airbyteJavaConnector { cdkVersionRequired = '0.21.0' features = ['db-destinations', 's3-destinations', 'typing-deduping'] - useLocalCdk = true + useLocalCdk = false } //remove once upgrading the CDK version to 0.4.x or later From ce5564f63d8024059a5a35c2d207bd20694b8233 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 21 Feb 2024 10:39:54 -0800 Subject: [PATCH 79/96] bump version --- .../airbyte-cdk/core/src/main/resources/version.properties | 2 +- .../destination-clickhouse-strict-encrypt/build.gradle | 4 ++-- .../connectors/destination-clickhouse/build.gradle | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties index 177f844ea34a..edf5dabc8075 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties @@ -1 +1 @@ -version=0.21.3 +version=0.22.0 diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle index abb7773e3bb6..4248850af8a1 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle @@ -4,9 +4,9 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.21.0' + cdkVersionRequired = '0.22.0' features = ['db-destinations', 's3-destinations', 'typing-deduping'] - useLocalCdk = false + useLocalCdk = true } //remove once upgrading the CDK version to 0.4.x or later diff --git a/airbyte-integrations/connectors/destination-clickhouse/build.gradle b/airbyte-integrations/connectors/destination-clickhouse/build.gradle index 3b921450798b..e93ab7ed0beb 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse/build.gradle @@ -4,9 +4,9 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.21.0' + cdkVersionRequired = '0.22.0' features = ['db-destinations', 's3-destinations', 'typing-deduping'] - useLocalCdk = false + useLocalCdk = true } //remove once upgrading the CDK version to 0.4.x or later From 3bec74e02bd8a2d2e8299745ca3784774140daff Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 21 Feb 2024 10:45:45 -0800 Subject: [PATCH 80/96] remove local version --- .../destination-clickhouse-strict-encrypt/build.gradle | 2 +- .../connectors/destination-clickhouse/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle index 4248850af8a1..8e4aa2ac9d2f 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle @@ -6,7 +6,7 @@ plugins { airbyteJavaConnector { cdkVersionRequired = '0.22.0' features = ['db-destinations', 's3-destinations', 'typing-deduping'] - useLocalCdk = true + useLocalCdk = false } //remove once upgrading the CDK version to 0.4.x or later diff --git a/airbyte-integrations/connectors/destination-clickhouse/build.gradle b/airbyte-integrations/connectors/destination-clickhouse/build.gradle index e93ab7ed0beb..629ecb3750aa 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse/build.gradle @@ -6,7 +6,7 @@ plugins { airbyteJavaConnector { cdkVersionRequired = '0.22.0' features = ['db-destinations', 's3-destinations', 'typing-deduping'] - useLocalCdk = true + useLocalCdk = false } //remove once upgrading the CDK version to 0.4.x or later From 376d1510630f947ffb86428471b81c4b4c743834 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 21 Feb 2024 13:43:48 -0800 Subject: [PATCH 81/96] use local again --- .../connectors/destination-clickhouse/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse/build.gradle b/airbyte-integrations/connectors/destination-clickhouse/build.gradle index 629ecb3750aa..e93ab7ed0beb 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse/build.gradle @@ -6,7 +6,7 @@ plugins { airbyteJavaConnector { cdkVersionRequired = '0.22.0' features = ['db-destinations', 's3-destinations', 'typing-deduping'] - useLocalCdk = false + useLocalCdk = true } //remove once upgrading the CDK version to 0.4.x or later From 504a79b0722d1b4bd23a3ffa438cecd82b8787cf Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Mon, 26 Feb 2024 15:41:51 -0800 Subject: [PATCH 82/96] formatting --- ...trictEncryptDestinationAcceptanceTest.java | 3 +- .../destination/oracle/OracleOperations.java | 37 +++++++++---------- .../NneOracleDestinationAcceptanceTest.java | 7 +--- .../SshOracleDestinationAcceptanceTest.java | 3 +- ...ryptedOracleDestinationAcceptanceTest.java | 26 ++++++------- 5 files changed, 37 insertions(+), 39 deletions(-) diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java index a37e41918ed1..2702b71b2ab9 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java @@ -113,7 +113,8 @@ protected List resolveIdentifier(final String identifier) { private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { - final String query = String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT.toUpperCase()); + final String query = + String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT.toUpperCase()); try (final DSLContext dslContext = getDslContext(config)) { final List result = getDatabase(dslContext).query(ctx -> ctx.fetch(query).stream().toList()); diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java index dd55caed976e..d213667fdfd7 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java @@ -62,20 +62,19 @@ public void createTableIfNotExists(final JdbcDatabase database, final String sch @Override public String createTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { return String.format( - """ - CREATE TABLE %s.%s ( - %s VARCHAR(64) PRIMARY KEY, - %s NCLOB, - %s TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP - %s TIMESTAMP WITH TIME ZONE DEFAULT NULL - ) - """, - schemaName, tableName, - upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID), - upperQuoted(JavaBaseConstants.COLUMN_NAME_DATA), - upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT), - upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT) - ); + """ + CREATE TABLE %s.%s ( + %s VARCHAR(64) PRIMARY KEY, + %s NCLOB, + %s TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP + %s TIMESTAMP WITH TIME ZONE DEFAULT NULL + ) + """, + schemaName, tableName, + upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID), + upperQuoted(JavaBaseConstants.COLUMN_NAME_DATA), + upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT), + upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT)); } private boolean tableExists(final JdbcDatabase database, final String schemaName, final String tableName) throws Exception { @@ -110,11 +109,10 @@ public void insertRecords(final JdbcDatabase database, throws Exception { final String tableName = String.format("%s.%s", schemaName, tempTableName); final String columns = String.format("(%s, %s, %s, %s)", - upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID), - upperQuoted(JavaBaseConstants.COLUMN_NAME_DATA), - upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT), - upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT) - ); + upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID), + upperQuoted(JavaBaseConstants.COLUMN_NAME_DATA), + upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT), + upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT)); insertRawRecordsInSingleQuery(tableName, columns, database, records, UUID::randomUUID); } @@ -185,4 +183,5 @@ public boolean isValidData(final JsonNode data) { public boolean isSchemaRequired() { return true; } + } diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java index 0a2d2daff0d9..599c50b76a5b 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java @@ -43,12 +43,9 @@ public void testEncryption() throws SQLException { String.format(DatabaseDriver.ORACLE.getUrlFormatString(), config.get("host").asText(), config.get("port").asInt(), - config.get("sid").asText() - ), + config.get("sid").asText()), getAdditionalProperties(algorithm), - Duration.ofMinutes(5) - ) - ); + Duration.ofMinutes(5))); final String networkServiceBanner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java index aa54a3348ed5..44221abcbd05 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java @@ -126,7 +126,8 @@ private List retrieveRecordsFromTable(final String tableName, final St (CheckedFunction, Exception>) mangledConfig -> getDatabaseFromConfig(mangledConfig) .query( ctx -> ctx - .fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT)))) + .fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, + upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT)))) .stream() .map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat())) .map(Jsons::deserialize) diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java index c86eff950713..180a0a543152 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java @@ -127,14 +127,14 @@ private List retrieveRecordsFromTable(final String tableName, final St throws SQLException { final DSLContext dslContext = getDSLContext(config); final List result = getDatabase(dslContext) - .query(ctx -> new ArrayList<>(ctx.fetch( - String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, - upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT))))); + .query(ctx -> new ArrayList<>(ctx.fetch( + String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, + upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT))))); return result - .stream() - .map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat())) - .map(Jsons::deserialize) - .collect(Collectors.toList()); + .stream() + .map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat())) + .map(Jsons::deserialize) + .collect(Collectors.toList()); } private static DSLContext getDSLContext(final JsonNode config) { @@ -161,13 +161,13 @@ protected void setup(final TestDestinationEnv testEnv, HashSet TEST_SCHE db.start(); config = getConfig(db); - final DSLContext dslContext = getDSLContext(config); - final Database database = getDatabase(dslContext); - database.query( - ctx -> ctx.fetch(String.format("CREATE USER %s IDENTIFIED BY %s", schemaName, schemaName))); - database.query(ctx -> ctx.fetch(String.format("GRANT ALL PRIVILEGES TO %s", schemaName))); + final DSLContext dslContext = getDSLContext(config); + final Database database = getDatabase(dslContext); + database.query( + ctx -> ctx.fetch(String.format("CREATE USER %s IDENTIFIED BY %s", schemaName, schemaName))); + database.query(ctx -> ctx.fetch(String.format("GRANT ALL PRIVILEGES TO %s", schemaName))); - ((ObjectNode) config).put(JdbcUtils.SCHEMA_KEY, dbName); + ((ObjectNode) config).put(JdbcUtils.SCHEMA_KEY, dbName); } @Override From 1a91597e9d9328215eaa7a7ee48f4e486ce190ef Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Mon, 26 Feb 2024 15:47:25 -0800 Subject: [PATCH 83/96] cdk version bump --- .../java/airbyte-cdk/core/src/main/resources/version.properties | 2 +- .../connectors/destination-oracle-strict-encrypt/build.gradle | 2 +- airbyte-integrations/connectors/destination-oracle/build.gradle | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties index b0d83063013b..ac44b08aea59 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties @@ -1 +1 @@ -version=0.23.2 +version=0.23.3 diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle index 3b2ffa30ce59..f18ae64b0186 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.21.0' + cdkVersionRequired = '0.23.3' features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = true } diff --git a/airbyte-integrations/connectors/destination-oracle/build.gradle b/airbyte-integrations/connectors/destination-oracle/build.gradle index 49eb4a08ad79..4e4163d2cf28 100644 --- a/airbyte-integrations/connectors/destination-oracle/build.gradle +++ b/airbyte-integrations/connectors/destination-oracle/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.21.0' + cdkVersionRequired = '0.23.3' features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = true } From 78794b88a4af6b67158adf275ba6aed39f13dbae Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Mon, 26 Feb 2024 17:44:22 -0800 Subject: [PATCH 84/96] appease the interface --- .../NoOpJdbcDestinationHandler.kt | 31 +++++++++++++++++++ .../destination/oracle/OracleDestination.java | 8 +++++ 2 files changed, 39 insertions(+) create mode 100644 airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/NoOpJdbcDestinationHandler.kt diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/NoOpJdbcDestinationHandler.kt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/NoOpJdbcDestinationHandler.kt new file mode 100644 index 000000000000..a83b789ab008 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/NoOpJdbcDestinationHandler.kt @@ -0,0 +1,31 @@ +package io.airbyte.cdk.integrations.destination.jdbc.typing_deduping + +import io.airbyte.cdk.db.jdbc.JdbcDatabase +import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType +import io.airbyte.integrations.base.destination.typing_deduping.DestinationInitialState +import io.airbyte.integrations.base.destination.typing_deduping.Sql +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig + +class NoOpJdbcDestinationHandler(databaseName: String, + jdbcDatabase: JdbcDatabase +): JdbcDestinationHandler(databaseName, jdbcDatabase) { + override fun execute(sql: Sql?) { + throw NotImplementedError("This JDBC Destination Handler does not support typing deduping") + } + + override fun gatherInitialState(streamConfigs: MutableList?): MutableList { + throw NotImplementedError("This JDBC Destination Handler does not support typing deduping") + } + + override fun existingSchemaMatchesStreamConfig( + stream: StreamConfig?, + existingTable: TableDefinition? + ): Boolean { + throw NotImplementedError("This JDBC Destination Handler does not support typing deduping") + } + + override fun toJdbcTypeName(airbyteType: AirbyteType?): String { + throw NotImplementedError("This JDBC Destination Handler does not support typing deduping") + } +} diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java index d83300f33777..a7daa5b8abbd 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java @@ -7,12 +7,15 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.cdk.integrations.base.Destination; import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination; import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcDestinationHandler; import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.NoOpJdbcDestinationHandler; import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.RawOnlySqlGenerator; import io.airbyte.commons.json.Jsons; import java.io.IOException; @@ -157,6 +160,11 @@ private static void runProcess(final String cmd, final Runtime run) throws IOExc } } + @Override + protected JdbcDestinationHandler getDestinationHandler(final String databaseName, final JdbcDatabase database) { + return new NoOpJdbcDestinationHandler(databaseName, database); + } + public static void main(final String[] args) throws Exception { final Destination destination = sshWrappedDestination(); LOGGER.info("starting destination: {}", OracleDestination.class); From dbcc23f7e777a9bae52330ffa2f35359f802dcd4 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Tue, 5 Mar 2024 14:03:09 -0800 Subject: [PATCH 85/96] formatting --- .../typing_deduping/NoOpJdbcDestinationHandler.kt | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/NoOpJdbcDestinationHandler.kt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/NoOpJdbcDestinationHandler.kt index a83b789ab008..519764ad5425 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/NoOpJdbcDestinationHandler.kt +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/NoOpJdbcDestinationHandler.kt @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.cdk.integrations.destination.jdbc.typing_deduping import io.airbyte.cdk.db.jdbc.JdbcDatabase @@ -7,14 +11,15 @@ import io.airbyte.integrations.base.destination.typing_deduping.DestinationIniti import io.airbyte.integrations.base.destination.typing_deduping.Sql import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig -class NoOpJdbcDestinationHandler(databaseName: String, - jdbcDatabase: JdbcDatabase -): JdbcDestinationHandler(databaseName, jdbcDatabase) { +class NoOpJdbcDestinationHandler(databaseName: String, jdbcDatabase: JdbcDatabase) : + JdbcDestinationHandler(databaseName, jdbcDatabase) { override fun execute(sql: Sql?) { throw NotImplementedError("This JDBC Destination Handler does not support typing deduping") } - override fun gatherInitialState(streamConfigs: MutableList?): MutableList { + override fun gatherInitialState( + streamConfigs: MutableList? + ): MutableList { throw NotImplementedError("This JDBC Destination Handler does not support typing deduping") } From 99585f27ad4bea5d66a6f680c492ce624e0f4656 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 6 Mar 2024 11:52:03 -0800 Subject: [PATCH 86/96] update kotlin file --- .../typing_deduping/NoOpJdbcDestinationHandler.kt | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/NoOpJdbcDestinationHandler.kt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/NoOpJdbcDestinationHandler.kt index 519764ad5425..67c1441e3215 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/NoOpJdbcDestinationHandler.kt +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/NoOpJdbcDestinationHandler.kt @@ -4,22 +4,21 @@ package io.airbyte.cdk.integrations.destination.jdbc.typing_deduping +import com.fasterxml.jackson.databind.JsonNode import io.airbyte.cdk.db.jdbc.JdbcDatabase import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType -import io.airbyte.integrations.base.destination.typing_deduping.DestinationInitialState import io.airbyte.integrations.base.destination.typing_deduping.Sql import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig +import org.jooq.SQLDialect -class NoOpJdbcDestinationHandler(databaseName: String, jdbcDatabase: JdbcDatabase) : - JdbcDestinationHandler(databaseName, jdbcDatabase) { +class NoOpJdbcDestinationHandler(databaseName: String, jdbcDatabase: JdbcDatabase, rawTableSchemaName: String, sqlDialect: SQLDialect) : + JdbcDestinationHandler(databaseName, jdbcDatabase, rawTableSchemaName, sqlDialect) { override fun execute(sql: Sql?) { throw NotImplementedError("This JDBC Destination Handler does not support typing deduping") } - override fun gatherInitialState( - streamConfigs: MutableList? - ): MutableList { + override fun toDestinationState(json: JsonNode?): DestinationState { throw NotImplementedError("This JDBC Destination Handler does not support typing deduping") } From cfca42166cf491ae2884de1b12a385eb256b92c5 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 6 Mar 2024 16:28:29 -0800 Subject: [PATCH 87/96] conform to other cdk updates --- .../destination/oracle/OracleDestination.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java index a7daa5b8abbd..7a65dd426ee4 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java @@ -18,6 +18,7 @@ import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.NoOpJdbcDestinationHandler; import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.RawOnlySqlGenerator; import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.destination.typing_deduping.migrators.MinimumDestinationState; import java.io.IOException; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; @@ -25,6 +26,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.RandomStringUtils; +import org.jooq.SQLDialect; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -141,6 +143,12 @@ protected JdbcSqlGenerator getSqlGenerator() { return new RawOnlySqlGenerator(new OracleNameTransformer()); } + @Override + protected JdbcDestinationHandler getDestinationHandler(final String databaseName, final JdbcDatabase database, + final String rawTableSchema) { + return new NoOpJdbcDestinationHandler<>(databaseName, database, rawTableSchema, SQLDialect.DEFAULT); + } + private static void convertAndImportCertificate(final String certificate) throws IOException, InterruptedException { final Runtime run = Runtime.getRuntime(); @@ -160,11 +168,6 @@ private static void runProcess(final String cmd, final Runtime run) throws IOExc } } - @Override - protected JdbcDestinationHandler getDestinationHandler(final String databaseName, final JdbcDatabase database) { - return new NoOpJdbcDestinationHandler(databaseName, database); - } - public static void main(final String[] args) throws Exception { final Destination destination = sshWrappedDestination(); LOGGER.info("starting destination: {}", OracleDestination.class); From ac6f714e7943f437467dce3f8bce07f8b26b9172 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 6 Mar 2024 16:32:52 -0800 Subject: [PATCH 88/96] version bump --- .../java/airbyte-cdk/core/src/main/resources/version.properties | 2 +- .../connectors/destination-oracle-strict-encrypt/build.gradle | 2 +- airbyte-integrations/connectors/destination-oracle/build.gradle | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties index e60bd207859a..ad61006b14c5 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties @@ -1 +1 @@ -version=0.23.15 +version=0.23.16 diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle index a77112730203..aa2fc8775b43 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.23.7' + cdkVersionRequired = '0.23.16' features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = true } diff --git a/airbyte-integrations/connectors/destination-oracle/build.gradle b/airbyte-integrations/connectors/destination-oracle/build.gradle index f71924a25a59..ef255fc7b219 100644 --- a/airbyte-integrations/connectors/destination-oracle/build.gradle +++ b/airbyte-integrations/connectors/destination-oracle/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.23.7' + cdkVersionRequired = '0.23.16' features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = true } From b686e3dfebf10df922f1c63cb5c49cc6dd66d58a Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 6 Mar 2024 17:34:21 -0800 Subject: [PATCH 89/96] appease gradle --- .../src/test/resources/expected_spec.json | 276 ++++++++++-------- ...trictEncryptDestinationAcceptanceTest.java | 36 ++- .../src/test/resources/expected_spec.json | 126 ++++++++ 3 files changed, 292 insertions(+), 146 deletions(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json index a64b1e0c31a1..0966582bcf4d 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json @@ -1,166 +1,188 @@ { - "documentationUrl": "https://docs.airbyte.com/integrations/destinations/clickhouse", - "supportsIncremental": true, - "supportsNormalization": true, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"], - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "ClickHouse Destination Spec", - "type": "object", - "required": ["host", "port", "database", "username"], - "additionalProperties": true, - "properties": { - "host": { - "title": "Host", - "description": "Hostname of the database.", - "type": "string", - "order": 0 + "documentationUrl" : "https://docs.airbyte.com/integrations/destinations/clickhouse", + "supportsIncremental" : true, + "supportsNormalization" : true, + "supportsDBT" : false, + "supported_destination_sync_modes" : [ + "overwrite", + "append" + ], + "connectionSpecification" : { + "$schema" : "http://json-schema.org/draft-07/schema#", + "title" : "ClickHouse Destination Spec", + "type" : "object", + "required" : [ + "host", + "port", + "database", + "username" + ], + "additionalProperties" : true, + "properties" : { + "host" : { + "title" : "Host", + "description" : "Hostname of the database.", + "type" : "string", + "order" : 0 }, - "port": { - "title": "Port", - "description": "HTTP port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 8123, - "examples": ["8123"], - "order": 1 + "port" : { + "title" : "Port", + "description" : "HTTP port of the database.", + "type" : "integer", + "minimum" : 0, + "maximum" : 65536, + "default" : 8123, + "examples" : [ + "8123" + ], + "order" : 1 }, - "database": { - "title": "DB Name", - "description": "Name of the database.", - "type": "string", - "order": 2 + "database" : { + "title" : "DB Name", + "description" : "Name of the database.", + "type" : "string", + "order" : 2 }, - "username": { - "title": "User", - "description": "Username to use to access the database.", - "type": "string", - "order": 3 + "username" : { + "title" : "User", + "description" : "Username to use to access the database.", + "type" : "string", + "order" : 3 }, - "password": { - "title": "Password", - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 4 + "password" : { + "title" : "Password", + "description" : "Password associated with the username.", + "type" : "string", + "airbyte_secret" : true, + "order" : 4 }, - "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "title": "JDBC URL Params", - "type": "string", - "order": 5 + "jdbc_url_params" : { + "description" : "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", + "title" : "JDBC URL Params", + "type" : "string", + "order" : 5 }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ + "raw_data_schema" : { + "type" : "string", + "description" : "The schema to write raw tables into (default: airbyte_internal)", + "title" : "Raw Table Schema Name", + "order" : 7 + }, + "tunnel_method" : { + "type" : "object", + "title" : "SSH Tunnel Method", + "description" : "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", + "oneOf" : [ { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 + "title" : "No Tunnel", + "required" : [ + "tunnel_method" + ], + "properties" : { + "tunnel_method" : { + "description" : "No ssh tunnel needed to connect to database", + "type" : "string", + "const" : "NO_TUNNEL", + "order" : 0 } } }, { - "title": "SSH Key Authentication", - "required": [ + "title" : "SSH Key Authentication", + "required" : [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "ssh_key" ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 + "properties" : { + "tunnel_method" : { + "description" : "Connect through a jump server tunnel host using username and ssh key", + "type" : "string", + "const" : "SSH_KEY_AUTH", + "order" : 0 }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 + "tunnel_host" : { + "title" : "SSH Tunnel Jump Server Host", + "description" : "Hostname of the jump server host that allows inbound ssh tunnel.", + "type" : "string", + "order" : 1 }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 + "tunnel_port" : { + "title" : "SSH Connection Port", + "description" : "Port on the proxy/jump server that accepts inbound ssh connections.", + "type" : "integer", + "minimum" : 0, + "maximum" : 65536, + "default" : 22, + "examples" : [ + "22" + ], + "order" : 2 }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 + "tunnel_user" : { + "title" : "SSH Login Username", + "description" : "OS-level username for logging into the jump server host.", + "type" : "string", + "order" : 3 }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 + "ssh_key" : { + "title" : "SSH Private Key", + "description" : "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", + "type" : "string", + "airbyte_secret" : true, + "multiline" : true, + "order" : 4 } } }, { - "title": "Password Authentication", - "required": [ + "title" : "Password Authentication", + "required" : [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "tunnel_user_password" ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 + "properties" : { + "tunnel_method" : { + "description" : "Connect through a jump server tunnel host using username and password authentication", + "type" : "string", + "const" : "SSH_PASSWORD_AUTH", + "order" : 0 }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 + "tunnel_host" : { + "title" : "SSH Tunnel Jump Server Host", + "description" : "Hostname of the jump server host that allows inbound ssh tunnel.", + "type" : "string", + "order" : 1 }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 + "tunnel_port" : { + "title" : "SSH Connection Port", + "description" : "Port on the proxy/jump server that accepts inbound ssh connections.", + "type" : "integer", + "minimum" : 0, + "maximum" : 65536, + "default" : 22, + "examples" : [ + "22" + ], + "order" : 2 }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 + "tunnel_user" : { + "title" : "SSH Login Username", + "description" : "OS-level username for logging into the jump server host", + "type" : "string", + "order" : 3 }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 + "tunnel_user_password" : { + "title" : "Password", + "description" : "OS-level password for logging into the jump server host", + "type" : "string", + "airbyte_secret" : true, + "order" : 4 } } } diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java index 2702b71b2ab9..a7cd54150050 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java @@ -24,6 +24,7 @@ import io.airbyte.commons.string.Strings; import io.airbyte.integrations.destination.oracle.OracleNameTransformer; import java.sql.SQLException; +import java.time.Duration; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -115,15 +116,13 @@ private List retrieveRecordsFromTable(final String tableName, final St throws SQLException { final String query = String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT.toUpperCase()); - - try (final DSLContext dslContext = getDslContext(config)) { - final List result = getDatabase(dslContext).query(ctx -> ctx.fetch(query).stream().toList()); - return result - .stream() - .map(r -> r.formatJSON(JSON_FORMAT)) - .map(Jsons::deserialize) - .collect(Collectors.toList()); - } + final DSLContext dslContext = getDslContext(config); + final List result = getDatabase(dslContext).query(ctx -> ctx.fetch(query).stream().toList()); + return result + .stream() + .map(r -> r.formatJSON(JSON_FORMAT)) + .map(Jsons::deserialize) + .collect(Collectors.toList()); } private static Database getDatabase(final DSLContext dslContext) { @@ -152,15 +151,13 @@ protected void setup(final TestDestinationEnv testEnv, final HashSet TES db.start(); config = getConfig(db); + final DSLContext dslContext = getDslContext(config); + final Database database = getDatabase(dslContext); + database.query( + ctx -> ctx.fetch(String.format("CREATE USER %s IDENTIFIED BY %s", schemaName, schemaName))); + database.query(ctx -> ctx.fetch(String.format("GRANT ALL PRIVILEGES TO %s", schemaName))); - try (final DSLContext dslContext = getDslContext(config)) { - final Database database = getDatabase(dslContext); - database.query( - ctx -> ctx.fetch(String.format("CREATE USER %s IDENTIFIED BY %s", schemaName, schemaName))); - database.query(ctx -> ctx.fetch(String.format("GRANT ALL PRIVILEGES TO %s", schemaName))); - - ((ObjectNode) config).put(JdbcUtils.SCHEMA_KEY, dbName); - } + ((ObjectNode) config).put(JdbcUtils.SCHEMA_KEY, dbName); } @Override @@ -183,7 +180,7 @@ public void testEncryption() throws SQLException { config.get(JdbcUtils.PORT_KEY).asInt(), config.get("sid").asText()), JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + - "oracle.net.encryption_types_client=( " + algorithm + " )", ";")); + "oracle.net.encryption_types_client=( " + algorithm + " )", ";"), Duration.ofMinutes(5)); final JdbcDatabase database = new DefaultJdbcDatabase(dataSource); final String networkServiceBanner = @@ -209,7 +206,8 @@ public void testCheckProtocol() throws SQLException { config.get(JdbcUtils.PORT_KEY).asInt(), config.get("sid").asText()), JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + - "oracle.net.encryption_types_client=( " + algorithm + " )", ";")); + "oracle.net.encryption_types_client=( " + algorithm + " )", ";"), + Duration.ofMinutes(5)); final JdbcDatabase database = new DefaultJdbcDatabase(dataSource); final String networkServiceBanner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual"; diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test/resources/expected_spec.json index 86b2da9a042e..26f5b5a7fed6 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test/resources/expected_spec.json @@ -59,6 +59,132 @@ "examples": ["airbyte"], "default": "airbyte", "order": 6 + }, + "raw_data_schema": { + "type": "string", + "description": "The schema to write raw tables into (default: airbyte_internal)", + "title": "Raw Table Schema Name", + "order": 7 + }, + "tunnel_method" : { + "type" : "object", + "title" : "SSH Tunnel Method", + "description" : "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", + "oneOf" : [ + { + "title" : "No Tunnel", + "required" : [ + "tunnel_method" + ], + "properties" : { + "tunnel_method" : { + "description" : "No ssh tunnel needed to connect to database", + "type" : "string", + "const" : "NO_TUNNEL", + "order" : 0 + } + } + }, + { + "title" : "SSH Key Authentication", + "required" : [ + "tunnel_method", + "tunnel_host", + "tunnel_port", + "tunnel_user", + "ssh_key" + ], + "properties" : { + "tunnel_method" : { + "description" : "Connect through a jump server tunnel host using username and ssh key", + "type" : "string", + "const" : "SSH_KEY_AUTH", + "order" : 0 + }, + "tunnel_host" : { + "title" : "SSH Tunnel Jump Server Host", + "description" : "Hostname of the jump server host that allows inbound ssh tunnel.", + "type" : "string", + "order" : 1 + }, + "tunnel_port" : { + "title" : "SSH Connection Port", + "description" : "Port on the proxy/jump server that accepts inbound ssh connections.", + "type" : "integer", + "minimum" : 0, + "maximum" : 65536, + "default" : 22, + "examples" : [ + "22" + ], + "order" : 2 + }, + "tunnel_user" : { + "title" : "SSH Login Username", + "description" : "OS-level username for logging into the jump server host.", + "type" : "string", + "order" : 3 + }, + "ssh_key" : { + "title" : "SSH Private Key", + "description" : "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", + "type" : "string", + "airbyte_secret" : true, + "multiline" : true, + "order" : 4 + } + } + }, + { + "title" : "Password Authentication", + "required" : [ + "tunnel_method", + "tunnel_host", + "tunnel_port", + "tunnel_user", + "tunnel_user_password" + ], + "properties" : { + "tunnel_method" : { + "description" : "Connect through a jump server tunnel host using username and password authentication", + "type" : "string", + "const" : "SSH_PASSWORD_AUTH", + "order" : 0 + }, + "tunnel_host" : { + "title" : "SSH Tunnel Jump Server Host", + "description" : "Hostname of the jump server host that allows inbound ssh tunnel.", + "type" : "string", + "order" : 1 + }, + "tunnel_port" : { + "title" : "SSH Connection Port", + "description" : "Port on the proxy/jump server that accepts inbound ssh connections.", + "type" : "integer", + "minimum" : 0, + "maximum" : 65536, + "default" : 22, + "examples" : [ + "22" + ], + "order" : 2 + }, + "tunnel_user" : { + "title" : "SSH Login Username", + "description" : "OS-level username for logging into the jump server host", + "type" : "string", + "order" : 3 + }, + "tunnel_user_password" : { + "title" : "Password", + "description" : "OS-level password for logging into the jump server host", + "type" : "string", + "airbyte_secret" : true, + "order" : 4 + } + } + } + ] } } } From 44738734b5966b7f864b081b0cdba8f450ba6260 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 6 Mar 2024 17:50:06 -0800 Subject: [PATCH 90/96] formatting --- .../NoOpJdbcDestinationHandler.kt | 14 +- .../src/test/resources/expected_spec.json | 280 +++++++++--------- ...trictEncryptDestinationAcceptanceTest.java | 5 +- .../src/test/resources/expected_spec.json | 166 +++++------ .../destination/oracle/OracleDestination.java | 3 +- 5 files changed, 229 insertions(+), 239 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/NoOpJdbcDestinationHandler.kt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/NoOpJdbcDestinationHandler.kt index 67c1441e3215..5e5edb140de1 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/NoOpJdbcDestinationHandler.kt +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/NoOpJdbcDestinationHandler.kt @@ -12,8 +12,18 @@ import io.airbyte.integrations.base.destination.typing_deduping.Sql import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig import org.jooq.SQLDialect -class NoOpJdbcDestinationHandler(databaseName: String, jdbcDatabase: JdbcDatabase, rawTableSchemaName: String, sqlDialect: SQLDialect) : - JdbcDestinationHandler(databaseName, jdbcDatabase, rawTableSchemaName, sqlDialect) { +class NoOpJdbcDestinationHandler( + databaseName: String, + jdbcDatabase: JdbcDatabase, + rawTableSchemaName: String, + sqlDialect: SQLDialect +) : + JdbcDestinationHandler( + databaseName, + jdbcDatabase, + rawTableSchemaName, + sqlDialect + ) { override fun execute(sql: Sql?) { throw NotImplementedError("This JDBC Destination Handler does not support typing deduping") } diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json index 0966582bcf4d..b2fd27f7028b 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json @@ -1,188 +1,172 @@ { - "documentationUrl" : "https://docs.airbyte.com/integrations/destinations/clickhouse", - "supportsIncremental" : true, - "supportsNormalization" : true, - "supportsDBT" : false, - "supported_destination_sync_modes" : [ - "overwrite", - "append" - ], - "connectionSpecification" : { - "$schema" : "http://json-schema.org/draft-07/schema#", - "title" : "ClickHouse Destination Spec", - "type" : "object", - "required" : [ - "host", - "port", - "database", - "username" - ], - "additionalProperties" : true, - "properties" : { - "host" : { - "title" : "Host", - "description" : "Hostname of the database.", - "type" : "string", - "order" : 0 + "documentationUrl": "https://docs.airbyte.com/integrations/destinations/clickhouse", + "supportsIncremental": true, + "supportsNormalization": true, + "supportsDBT": false, + "supported_destination_sync_modes": ["overwrite", "append"], + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ClickHouse Destination Spec", + "type": "object", + "required": ["host", "port", "database", "username"], + "additionalProperties": true, + "properties": { + "host": { + "title": "Host", + "description": "Hostname of the database.", + "type": "string", + "order": 0 }, - "port" : { - "title" : "Port", - "description" : "HTTP port of the database.", - "type" : "integer", - "minimum" : 0, - "maximum" : 65536, - "default" : 8123, - "examples" : [ - "8123" - ], - "order" : 1 + "port": { + "title": "Port", + "description": "HTTP port of the database.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 8123, + "examples": ["8123"], + "order": 1 }, - "database" : { - "title" : "DB Name", - "description" : "Name of the database.", - "type" : "string", - "order" : 2 + "database": { + "title": "DB Name", + "description": "Name of the database.", + "type": "string", + "order": 2 }, - "username" : { - "title" : "User", - "description" : "Username to use to access the database.", - "type" : "string", - "order" : 3 + "username": { + "title": "User", + "description": "Username to use to access the database.", + "type": "string", + "order": 3 }, - "password" : { - "title" : "Password", - "description" : "Password associated with the username.", - "type" : "string", - "airbyte_secret" : true, - "order" : 4 + "password": { + "title": "Password", + "description": "Password associated with the username.", + "type": "string", + "airbyte_secret": true, + "order": 4 }, - "jdbc_url_params" : { - "description" : "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "title" : "JDBC URL Params", - "type" : "string", - "order" : 5 + "jdbc_url_params": { + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", + "title": "JDBC URL Params", + "type": "string", + "order": 5 }, - "raw_data_schema" : { - "type" : "string", - "description" : "The schema to write raw tables into (default: airbyte_internal)", - "title" : "Raw Table Schema Name", - "order" : 7 + "raw_data_schema": { + "type": "string", + "description": "The schema to write raw tables into (default: airbyte_internal)", + "title": "Raw Table Schema Name", + "order": 7 }, - "tunnel_method" : { - "type" : "object", - "title" : "SSH Tunnel Method", - "description" : "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf" : [ + "tunnel_method": { + "type": "object", + "title": "SSH Tunnel Method", + "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", + "oneOf": [ { - "title" : "No Tunnel", - "required" : [ - "tunnel_method" - ], - "properties" : { - "tunnel_method" : { - "description" : "No ssh tunnel needed to connect to database", - "type" : "string", - "const" : "NO_TUNNEL", - "order" : 0 + "title": "No Tunnel", + "required": ["tunnel_method"], + "properties": { + "tunnel_method": { + "description": "No ssh tunnel needed to connect to database", + "type": "string", + "const": "NO_TUNNEL", + "order": 0 } } }, { - "title" : "SSH Key Authentication", - "required" : [ + "title": "SSH Key Authentication", + "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "ssh_key" ], - "properties" : { - "tunnel_method" : { - "description" : "Connect through a jump server tunnel host using username and ssh key", - "type" : "string", - "const" : "SSH_KEY_AUTH", - "order" : 0 + "properties": { + "tunnel_method": { + "description": "Connect through a jump server tunnel host using username and ssh key", + "type": "string", + "const": "SSH_KEY_AUTH", + "order": 0 }, - "tunnel_host" : { - "title" : "SSH Tunnel Jump Server Host", - "description" : "Hostname of the jump server host that allows inbound ssh tunnel.", - "type" : "string", - "order" : 1 + "tunnel_host": { + "title": "SSH Tunnel Jump Server Host", + "description": "Hostname of the jump server host that allows inbound ssh tunnel.", + "type": "string", + "order": 1 }, - "tunnel_port" : { - "title" : "SSH Connection Port", - "description" : "Port on the proxy/jump server that accepts inbound ssh connections.", - "type" : "integer", - "minimum" : 0, - "maximum" : 65536, - "default" : 22, - "examples" : [ - "22" - ], - "order" : 2 + "tunnel_port": { + "title": "SSH Connection Port", + "description": "Port on the proxy/jump server that accepts inbound ssh connections.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 22, + "examples": ["22"], + "order": 2 }, - "tunnel_user" : { - "title" : "SSH Login Username", - "description" : "OS-level username for logging into the jump server host.", - "type" : "string", - "order" : 3 + "tunnel_user": { + "title": "SSH Login Username", + "description": "OS-level username for logging into the jump server host.", + "type": "string", + "order": 3 }, - "ssh_key" : { - "title" : "SSH Private Key", - "description" : "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type" : "string", - "airbyte_secret" : true, - "multiline" : true, - "order" : 4 + "ssh_key": { + "title": "SSH Private Key", + "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", + "type": "string", + "airbyte_secret": true, + "multiline": true, + "order": 4 } } }, { - "title" : "Password Authentication", - "required" : [ + "title": "Password Authentication", + "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "tunnel_user_password" ], - "properties" : { - "tunnel_method" : { - "description" : "Connect through a jump server tunnel host using username and password authentication", - "type" : "string", - "const" : "SSH_PASSWORD_AUTH", - "order" : 0 + "properties": { + "tunnel_method": { + "description": "Connect through a jump server tunnel host using username and password authentication", + "type": "string", + "const": "SSH_PASSWORD_AUTH", + "order": 0 }, - "tunnel_host" : { - "title" : "SSH Tunnel Jump Server Host", - "description" : "Hostname of the jump server host that allows inbound ssh tunnel.", - "type" : "string", - "order" : 1 + "tunnel_host": { + "title": "SSH Tunnel Jump Server Host", + "description": "Hostname of the jump server host that allows inbound ssh tunnel.", + "type": "string", + "order": 1 }, - "tunnel_port" : { - "title" : "SSH Connection Port", - "description" : "Port on the proxy/jump server that accepts inbound ssh connections.", - "type" : "integer", - "minimum" : 0, - "maximum" : 65536, - "default" : 22, - "examples" : [ - "22" - ], - "order" : 2 + "tunnel_port": { + "title": "SSH Connection Port", + "description": "Port on the proxy/jump server that accepts inbound ssh connections.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 22, + "examples": ["22"], + "order": 2 }, - "tunnel_user" : { - "title" : "SSH Login Username", - "description" : "OS-level username for logging into the jump server host", - "type" : "string", - "order" : 3 + "tunnel_user": { + "title": "SSH Login Username", + "description": "OS-level username for logging into the jump server host", + "type": "string", + "order": 3 }, - "tunnel_user_password" : { - "title" : "Password", - "description" : "OS-level password for logging into the jump server host", - "type" : "string", - "airbyte_secret" : true, - "order" : 4 + "tunnel_user_password": { + "title": "Password", + "description": "OS-level password for logging into the jump server host", + "type": "string", + "airbyte_secret": true, + "order": 4 } } } diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java index a7cd54150050..20cb8a31c8d0 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java @@ -180,7 +180,8 @@ public void testEncryption() throws SQLException { config.get(JdbcUtils.PORT_KEY).asInt(), config.get("sid").asText()), JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + - "oracle.net.encryption_types_client=( " + algorithm + " )", ";"), Duration.ofMinutes(5)); + "oracle.net.encryption_types_client=( " + algorithm + " )", ";"), + Duration.ofMinutes(5)); final JdbcDatabase database = new DefaultJdbcDatabase(dataSource); final String networkServiceBanner = @@ -207,7 +208,7 @@ public void testCheckProtocol() throws SQLException { config.get("sid").asText()), JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + "oracle.net.encryption_types_client=( " + algorithm + " )", ";"), - Duration.ofMinutes(5)); + Duration.ofMinutes(5)); final JdbcDatabase database = new DefaultJdbcDatabase(dataSource); final String networkServiceBanner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual"; diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test/resources/expected_spec.json index 26f5b5a7fed6..b6badb3b7228 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test/resources/expected_spec.json @@ -66,121 +66,115 @@ "title": "Raw Table Schema Name", "order": 7 }, - "tunnel_method" : { - "type" : "object", - "title" : "SSH Tunnel Method", - "description" : "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf" : [ + "tunnel_method": { + "type": "object", + "title": "SSH Tunnel Method", + "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", + "oneOf": [ { - "title" : "No Tunnel", - "required" : [ - "tunnel_method" - ], - "properties" : { - "tunnel_method" : { - "description" : "No ssh tunnel needed to connect to database", - "type" : "string", - "const" : "NO_TUNNEL", - "order" : 0 + "title": "No Tunnel", + "required": ["tunnel_method"], + "properties": { + "tunnel_method": { + "description": "No ssh tunnel needed to connect to database", + "type": "string", + "const": "NO_TUNNEL", + "order": 0 } } }, { - "title" : "SSH Key Authentication", - "required" : [ + "title": "SSH Key Authentication", + "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "ssh_key" ], - "properties" : { - "tunnel_method" : { - "description" : "Connect through a jump server tunnel host using username and ssh key", - "type" : "string", - "const" : "SSH_KEY_AUTH", - "order" : 0 + "properties": { + "tunnel_method": { + "description": "Connect through a jump server tunnel host using username and ssh key", + "type": "string", + "const": "SSH_KEY_AUTH", + "order": 0 }, - "tunnel_host" : { - "title" : "SSH Tunnel Jump Server Host", - "description" : "Hostname of the jump server host that allows inbound ssh tunnel.", - "type" : "string", - "order" : 1 + "tunnel_host": { + "title": "SSH Tunnel Jump Server Host", + "description": "Hostname of the jump server host that allows inbound ssh tunnel.", + "type": "string", + "order": 1 }, - "tunnel_port" : { - "title" : "SSH Connection Port", - "description" : "Port on the proxy/jump server that accepts inbound ssh connections.", - "type" : "integer", - "minimum" : 0, - "maximum" : 65536, - "default" : 22, - "examples" : [ - "22" - ], - "order" : 2 + "tunnel_port": { + "title": "SSH Connection Port", + "description": "Port on the proxy/jump server that accepts inbound ssh connections.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 22, + "examples": ["22"], + "order": 2 }, - "tunnel_user" : { - "title" : "SSH Login Username", - "description" : "OS-level username for logging into the jump server host.", - "type" : "string", - "order" : 3 + "tunnel_user": { + "title": "SSH Login Username", + "description": "OS-level username for logging into the jump server host.", + "type": "string", + "order": 3 }, - "ssh_key" : { - "title" : "SSH Private Key", - "description" : "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type" : "string", - "airbyte_secret" : true, - "multiline" : true, - "order" : 4 + "ssh_key": { + "title": "SSH Private Key", + "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", + "type": "string", + "airbyte_secret": true, + "multiline": true, + "order": 4 } } }, { - "title" : "Password Authentication", - "required" : [ + "title": "Password Authentication", + "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "tunnel_user_password" ], - "properties" : { - "tunnel_method" : { - "description" : "Connect through a jump server tunnel host using username and password authentication", - "type" : "string", - "const" : "SSH_PASSWORD_AUTH", - "order" : 0 + "properties": { + "tunnel_method": { + "description": "Connect through a jump server tunnel host using username and password authentication", + "type": "string", + "const": "SSH_PASSWORD_AUTH", + "order": 0 }, - "tunnel_host" : { - "title" : "SSH Tunnel Jump Server Host", - "description" : "Hostname of the jump server host that allows inbound ssh tunnel.", - "type" : "string", - "order" : 1 + "tunnel_host": { + "title": "SSH Tunnel Jump Server Host", + "description": "Hostname of the jump server host that allows inbound ssh tunnel.", + "type": "string", + "order": 1 }, - "tunnel_port" : { - "title" : "SSH Connection Port", - "description" : "Port on the proxy/jump server that accepts inbound ssh connections.", - "type" : "integer", - "minimum" : 0, - "maximum" : 65536, - "default" : 22, - "examples" : [ - "22" - ], - "order" : 2 + "tunnel_port": { + "title": "SSH Connection Port", + "description": "Port on the proxy/jump server that accepts inbound ssh connections.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 22, + "examples": ["22"], + "order": 2 }, - "tunnel_user" : { - "title" : "SSH Login Username", - "description" : "OS-level username for logging into the jump server host", - "type" : "string", - "order" : 3 + "tunnel_user": { + "title": "SSH Login Username", + "description": "OS-level username for logging into the jump server host", + "type": "string", + "order": 3 }, - "tunnel_user_password" : { - "title" : "Password", - "description" : "OS-level password for logging into the jump server host", - "type" : "string", - "airbyte_secret" : true, - "order" : 4 + "tunnel_user_password": { + "title": "Password", + "description": "OS-level password for logging into the jump server host", + "type": "string", + "airbyte_secret": true, + "order": 4 } } } diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java index 7a65dd426ee4..39f9d9c13353 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java @@ -144,7 +144,8 @@ protected JdbcSqlGenerator getSqlGenerator() { } @Override - protected JdbcDestinationHandler getDestinationHandler(final String databaseName, final JdbcDatabase database, + protected JdbcDestinationHandler getDestinationHandler(final String databaseName, + final JdbcDatabase database, final String rawTableSchema) { return new NoOpJdbcDestinationHandler<>(databaseName, database, rawTableSchema, SQLDialect.DEFAULT); } From 056407e5477a0a0d61752e36e6c58336e4829803 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 6 Mar 2024 20:13:30 -0800 Subject: [PATCH 91/96] add always type and dedupe method --- .../destination/jdbc/AbstractJdbcDestination.java | 15 ++++++++++++++- .../destination/oracle/OracleDestination.java | 5 +++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java index 86935bbe2937..36d8856bef0b 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java @@ -77,6 +77,15 @@ protected String getConfigSchemaKey() { return "schema"; } + /** + * If the destination should always disable type dedupe, override this method to return true. + * We only type and dedupe if we create final tables. + * @return whether the destination should always disable type dedupe + */ + protected boolean shouldAlwaysDisableTypeDedupe() { + return false; + } + public AbstractJdbcDestination(final String driverClass, final NamingConventionTransformer namingResolver, final SqlOperations sqlOperations) { @@ -323,7 +332,7 @@ private TyperDeduper getV2TyperDeduper(final JsonNode config, final ConfiguredAi final NoopV2TableMigrator v2TableMigrator = new NoopV2TableMigrator(); final DestinationHandler destinationHandler = getDestinationHandler(databaseName, database, rawNamespaceOverride.orElse(DEFAULT_AIRBYTE_INTERNAL_NAMESPACE)); - final boolean disableTypeDedupe = config.has(DISABLE_TYPE_DEDUPE) && config.get(DISABLE_TYPE_DEDUPE).asBoolean(false); + final boolean disableTypeDedupe = isTypeDedupeDisabled(config); final TyperDeduper typerDeduper; if (disableTypeDedupe) { typerDeduper = new NoOpTyperDeduperWithV1V2Migrations<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator, List.of()); @@ -334,4 +343,8 @@ private TyperDeduper getV2TyperDeduper(final JsonNode config, final ConfiguredAi return typerDeduper; } + private boolean isTypeDedupeDisabled(final JsonNode config) { + return (config.has(DISABLE_TYPE_DEDUPE) && config.get(DISABLE_TYPE_DEDUPE).asBoolean(false)) || shouldAlwaysDisableTypeDedupe(); + } + } diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java index 39f9d9c13353..385bb0af7447 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java @@ -138,6 +138,11 @@ public boolean isV2Destination() { return true; } + @Override + protected boolean shouldAlwaysDisableTypeDedupe() { + return true; + } + @Override protected JdbcSqlGenerator getSqlGenerator() { return new RawOnlySqlGenerator(new OracleNameTransformer()); From 7f9777bf8c42c4a87801a599e78eb9a2f7b6d664 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 6 Mar 2024 20:15:51 -0800 Subject: [PATCH 92/96] formatting --- .../destination/jdbc/AbstractJdbcDestination.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java index 36d8856bef0b..89d1b01ccb30 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java @@ -78,8 +78,9 @@ protected String getConfigSchemaKey() { } /** - * If the destination should always disable type dedupe, override this method to return true. - * We only type and dedupe if we create final tables. + * If the destination should always disable type dedupe, override this method to return true. We + * only type and dedupe if we create final tables. + * * @return whether the destination should always disable type dedupe */ protected boolean shouldAlwaysDisableTypeDedupe() { From 898cbd94b87cbbbf57d860c6008367b1e473b092 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Thu, 7 Mar 2024 16:44:05 -0800 Subject: [PATCH 93/96] test timeouts --- .../destination-oracle-strict-encrypt/gradle.properties | 1 + .../connectors/destination-oracle/gradle.properties | 1 + 2 files changed, 2 insertions(+) create mode 100644 airbyte-integrations/connectors/destination-oracle-strict-encrypt/gradle.properties create mode 100644 airbyte-integrations/connectors/destination-oracle/gradle.properties diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/gradle.properties b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/gradle.properties new file mode 100644 index 000000000000..6a0ea27448c4 --- /dev/null +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/gradle.properties @@ -0,0 +1 @@ +junitMethodExecutionTimeout = 30 m diff --git a/airbyte-integrations/connectors/destination-oracle/gradle.properties b/airbyte-integrations/connectors/destination-oracle/gradle.properties new file mode 100644 index 000000000000..6a0ea27448c4 --- /dev/null +++ b/airbyte-integrations/connectors/destination-oracle/gradle.properties @@ -0,0 +1 @@ +junitMethodExecutionTimeout = 30 m From 0664a96265c9c7c7fa2d4930968c08d919427d70 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Mon, 11 Mar 2024 07:32:16 -0700 Subject: [PATCH 94/96] add airbyte meta column to raw table --- .../integrations/destination/oracle/OracleOperations.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java index d213667fdfd7..75979e3888b1 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java @@ -66,15 +66,17 @@ public String createTableQuery(final JdbcDatabase database, final String schemaN CREATE TABLE %s.%s ( %s VARCHAR(64) PRIMARY KEY, %s NCLOB, - %s TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP - %s TIMESTAMP WITH TIME ZONE DEFAULT NULL + %s TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + %s TIMESTAMP WITH TIME ZONE DEFAULT NULL, + %s NCLOB ) """, schemaName, tableName, upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID), upperQuoted(JavaBaseConstants.COLUMN_NAME_DATA), upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT), - upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT)); + upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT), + upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_META)); } private boolean tableExists(final JdbcDatabase database, final String schemaName, final String tableName) throws Exception { From 3b3f819e0fd59854f89407b9313ae72d58208e61 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Mon, 11 Mar 2024 08:24:35 -0700 Subject: [PATCH 95/96] add airbyte meta column --- .../destination/oracle/OracleOperations.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java index 75979e3888b1..7f673077a58a 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java @@ -12,7 +12,6 @@ import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; -import io.airbyte.commons.json.Jsons; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Timestamp; @@ -65,10 +64,10 @@ public String createTableQuery(final JdbcDatabase database, final String schemaN """ CREATE TABLE %s.%s ( %s VARCHAR(64) PRIMARY KEY, - %s NCLOB, + %s JSON, %s TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, %s TIMESTAMP WITH TIME ZONE DEFAULT NULL, - %s NCLOB + %s JSON ) """, schemaName, tableName, @@ -110,11 +109,12 @@ public void insertRecords(final JdbcDatabase database, final String tempTableName) throws Exception { final String tableName = String.format("%s.%s", schemaName, tempTableName); - final String columns = String.format("(%s, %s, %s, %s)", + final String columns = String.format("(%s, %s, %s, %s, %s)", upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID), upperQuoted(JavaBaseConstants.COLUMN_NAME_DATA), upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT), - upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT)); + upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT), + upperQuoted(JavaBaseConstants.COLUMN_NAME_AB_META)); insertRawRecordsInSingleQuery(tableName, columns, database, records, UUID::randomUUID); } @@ -152,9 +152,11 @@ private static void insertRawRecordsInSingleQuery(final String tableName, // 1-indexed final JsonNode formattedData = StandardNameTransformer.formatJsonPath(message.getRecord().getData()); statement.setString(i++, uuidSupplier.get().toString()); - statement.setString(i++, Jsons.serialize(formattedData)); + statement.setObject(i++, formattedData); +// statement.setString(i++, Jsons.serialize(formattedData)); statement.setTimestamp(i++, Timestamp.from(Instant.ofEpochMilli(message.getRecord().getEmittedAt()))); statement.setNull(i++, Types.TIMESTAMP); + statement.setObject(i++, ""); } statement.execute(); From 561c444d77c0dc0ebae1f0bc84562ae927e075d0 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Mon, 11 Mar 2024 08:26:04 -0700 Subject: [PATCH 96/96] formatting --- .../integrations/destination/oracle/OracleOperations.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java index 7f673077a58a..60f197dc9801 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java @@ -153,7 +153,7 @@ private static void insertRawRecordsInSingleQuery(final String tableName, final JsonNode formattedData = StandardNameTransformer.formatJsonPath(message.getRecord().getData()); statement.setString(i++, uuidSupplier.get().toString()); statement.setObject(i++, formattedData); -// statement.setString(i++, Jsons.serialize(formattedData)); + // statement.setString(i++, Jsons.serialize(formattedData)); statement.setTimestamp(i++, Timestamp.from(Instant.ofEpochMilli(message.getRecord().getEmittedAt()))); statement.setNull(i++, Types.TIMESTAMP); statement.setObject(i++, "");