From 6d8a3a2bc4f4ca79f10164447a90fdce5c9ad6f9 Mon Sep 17 00:00:00 2001 From: Jonathan Pearlin <jonathan@airbyte.io> Date: Tue, 17 Dec 2024 10:46:53 -0500 Subject: [PATCH 1/7] fix: return correct language tag when using Kotlin build script (#49835) --- .../connectors_qa/src/connectors_qa/checks/metadata.py | 4 +++- .../connectors/connectors_qa/src/connectors_qa/consts.py | 1 + airbyte-ci/connectors/pipelines/README.md | 1 + airbyte-ci/connectors/pipelines/pyproject.toml | 2 +- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/metadata.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/metadata.py index d06a7945ef5c..da2b73b25c72 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/metadata.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/metadata.py @@ -48,7 +48,9 @@ def get_expected_language_tag(self, connector: Connector) -> str: connector.code_directory / consts.PYPROJECT_FILE_NAME ).exists(): return self.PYTHON_LANGUAGE_TAG - elif (connector.code_directory / consts.GRADLE_FILE_NAME).exists(): + elif (connector.code_directory / consts.GRADLE_FILE_NAME).exists() or ( + connector.code_directory / consts.GRADLE_KOTLIN_FILE_NAME + ).exists(): return self.JAVA_LANGUAGE_TAG else: raise ValueError("Could not infer the language tag from the connector directory") diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/consts.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/consts.py index ba17a607cc18..2d4d863aab65 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/consts.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/consts.py @@ -9,6 +9,7 @@ DOCKERFILE_NAME = "Dockerfile" DOCUMENTATION_STANDARDS_URL = "https://hackmd.io/Bz75cgATSbm7DjrAqgl4rw" GRADLE_FILE_NAME = "build.gradle" +GRADLE_KOTLIN_FILE_NAME = "build.gradle.kts" LICENSE_FAQ_URL = "https://docs.airbyte.com/developer-guides/licenses/license-faq" LOW_CODE_MANIFEST_FILE_NAME = "manifest.yaml" METADATA_DOCUMENTATION_URL = "https://docs.airbyte.com/connector-development/connector-metadata-file" diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index ec4f0875090a..c9206c9b9009 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -854,6 +854,7 @@ airbyte-ci connectors --language=low-code migrate-to-manifest-only | Version | PR | Description | |---------|------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------| +| 4.46.5 | [#49835](https://github.com/airbytehq/airbyte/pull/49835) | Fix connector language discovery for projects with Kotlin Gradle build scripts. | | 4.46.4 | [#49462](https://github.com/airbytehq/airbyte/pull/49462) | Support Kotlin Gradle build scripts in connectors. | | 4.46.3 | [#49465](https://github.com/airbytehq/airbyte/pull/49465) | Fix `--use-local-cdk` on rootless connectors. | | 4.46.2 | [#49136](https://github.com/airbytehq/airbyte/pull/49136) | Fix failed install of python components due to non-root permissions. | diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 92a0aee6df37..dc32ae4df374 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "4.46.4" +version = "4.46.5" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte <contact@airbyte.io>"] From 4dc13b301c81f850a23e95e54f816bb3f291ae39 Mon Sep 17 00:00:00 2001 From: Jonathan Pearlin <jonathan@airbyte.io> Date: Tue, 17 Dec 2024 11:13:47 -0500 Subject: [PATCH 2/7] chore: Skeleton project for destination MSSQL V2 (#49460) --- .../connectors/destination-mssql-v2/README.md | 91 +++++++++++++++++++ .../destination-mssql-v2/build.gradle.kts | 52 +++++++++++ .../destination-mssql-v2/gradle.properties | 1 + .../connectors/destination-mssql-v2/icon.svg | 1 + .../destination-mssql-v2/metadata.yaml | 34 +++++++ .../destination/mssql/v2/MSSQLDestination.kt | 14 +++ .../mssql/v2/config/MSSQLConfiguration.kt | 28 ++++++ .../mssql/v2/config/MSSQLSpecification.kt | 26 ++++++ docs/integrations/destinations/mssql-v2.md | 12 +++ 9 files changed, 259 insertions(+) create mode 100644 airbyte-integrations/connectors/destination-mssql-v2/README.md create mode 100644 airbyte-integrations/connectors/destination-mssql-v2/build.gradle.kts create mode 100644 airbyte-integrations/connectors/destination-mssql-v2/gradle.properties create mode 100644 airbyte-integrations/connectors/destination-mssql-v2/icon.svg create mode 100644 airbyte-integrations/connectors/destination-mssql-v2/metadata.yaml create mode 100644 airbyte-integrations/connectors/destination-mssql-v2/src/main/kotlin/io/airbyte/integrations/destination/mssql/v2/MSSQLDestination.kt create mode 100644 airbyte-integrations/connectors/destination-mssql-v2/src/main/kotlin/io/airbyte/integrations/destination/mssql/v2/config/MSSQLConfiguration.kt create mode 100644 airbyte-integrations/connectors/destination-mssql-v2/src/main/kotlin/io/airbyte/integrations/destination/mssql/v2/config/MSSQLSpecification.kt create mode 100644 docs/integrations/destinations/mssql-v2.md diff --git a/airbyte-integrations/connectors/destination-mssql-v2/README.md b/airbyte-integrations/connectors/destination-mssql-v2/README.md new file mode 100644 index 000000000000..4f2fe1572dfa --- /dev/null +++ b/airbyte-integrations/connectors/destination-mssql-v2/README.md @@ -0,0 +1,91 @@ +# Microsoft SQL Server V2 (Bulk CDK) Destination + +## Build + +### airbyte-ci + +To build the connector via the [Airbyte CI CLI tool](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md), navigate to the root of the [Airbyte repository](https://github.com/airbytehq/airbyte) and execute the following command: + +```shell +> airbyte-ci connectors --name=destination-mssql-v2 build +``` + +### Gradle + +To build the connector via [Gradle](https://gradle.org/), navigate to the root of the [Airbyte repository](https://github.com/airbytehq/airbyte) and execute the following command: + +```shell +> ./gradlew :airbyte-integrations:connectors:destination-mssql-v2:build +``` +## Execute + +### Local Execution via Docker + +In order to run the connector image locally, first either build the connector's [Docker](https://www.docker.com/) image using the commands found +in this section of this document OR build the image using the following command: + +```shell +> ./gradlew :airbyte-integrations:connectors:destination-mssql-v2:buildConnectorImage +``` + +The built image will automatically be tagged with the `dev` label. To run the connector image, use the following commands: + +```shell +docker run --rm airbyte/destination-mssql-v2:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-mssql-v2:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-mssql-v2:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-mssql-v2:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +## Test + +The connector contains both unit and acceptance tests which can each be executed from the local environment. + +### Unit Tests + +The connector uses a combination of [Kotlin](https://kotlinlang.org/), [JUnit 5](https://junit.org/junit5/) and [MockK](https://mockk.io/) +to implement unit tests. Existing tests can be found within the destination-mssql-v2 module in the conventional `src/test/kotlin` source folder. New tests should also be added to this location. + +The unit tests can be executed either via the [Airbyte CI CLI tool](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) or [Gradle](https://gradle.org/): + +###### Airbyte CI CLI +```shell +> airbyte-ci connectors --name=destination-mssql-v2 test +``` + +###### Gradle +```shell +> ./gradlew :airbyte-integrations:connectors:destination-mssql-v2:test +``` + +### Acceptance Tests + +The [Airbyte project](https://github.com/airbytehq/airbyte) a standard test suite that all destination connectors must pass. The tests require specific implementations of a few components in order to connect the acceptance test suite with the connector's specific logic. The existing acceptance test scaffolding can be found in the conventional `src/test-integration/kotlin` source folder. + +The acceptance tests can be executed either via the [Airbyte CI CLI tool](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) or [Gradle](https://gradle.org/): + +###### Airbyte CI CLI +```shell +> airbyte-ci connectors --name=destination-mssql-v2 test +``` + +###### Gradle +```shell +> ./gradlew :airbyte-integrations:connectors:destination-mssql-v2:integrationTest +``` + +## Release + +### Publishing a new version of the connector + +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? + +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-mssql-v2 test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/mssql-v2.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + + diff --git a/airbyte-integrations/connectors/destination-mssql-v2/build.gradle.kts b/airbyte-integrations/connectors/destination-mssql-v2/build.gradle.kts new file mode 100644 index 000000000000..4cc0467ff7bd --- /dev/null +++ b/airbyte-integrations/connectors/destination-mssql-v2/build.gradle.kts @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +plugins { + id("application") + id("airbyte-bulk-connector") +} + +airbyteBulkConnector { + core = "load" + toolkits = listOf() + cdk = "local" +} + +application { + mainClass = "io.airbyte.integrations.destination.mssql.v2.MSSQLDestination" + + applicationDefaultJvmArgs = listOf("-XX:+ExitOnOutOfMemoryError", "-XX:MaxRAMPercentage=75.0") + + // Uncomment and replace to run locally + //applicationDefaultJvmArgs = listOf("-XX:+ExitOnOutOfMemoryError", "-XX:MaxRAMPercentage=75.0", "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED", "--add-opens", "java.base/sun.security.action=ALL-UNNAMED", "--add-opens", "java.base/java.lang=ALL-UNNAMED") +} + +val junitVersion = "5.11.3" + +configurations.configureEach { + // Exclude additional SLF4J providers from all classpaths + exclude(mapOf("group" to "org.slf4j", "module" to "slf4j-reload4j")) +} + +// Uncomment to run locally +//tasks.run.configure { +// standardInput = System.`in` +//} + +dependencies { + implementation("com.microsoft.sqlserver:mssql-jdbc:12.8.1.jre11") + implementation("io.github.oshai:kotlin-logging-jvm:7.0.0") + implementation("jakarta.inject:jakarta.inject-api:2.0.1") + implementation("com.github.spotbugs:spotbugs-annotations:4.8.6") + implementation("io.micronaut:micronaut-inject:4.6.1") + + testImplementation("io.mockk:mockk:1.13.13") + testImplementation("org.junit.jupiter:junit-jupiter-api:$junitVersion") + testImplementation("org.junit.jupiter:junit-jupiter-params:$junitVersion") + testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:$junitVersion") +} + +tasks.named<Test>("test") { + systemProperties(mapOf("mockk.junit.extension.keepmocks" to "true", "mockk.junit.extension.requireParallelTesting" to "true")) +} diff --git a/airbyte-integrations/connectors/destination-mssql-v2/gradle.properties b/airbyte-integrations/connectors/destination-mssql-v2/gradle.properties new file mode 100644 index 000000000000..4dbe8b8729df --- /dev/null +++ b/airbyte-integrations/connectors/destination-mssql-v2/gradle.properties @@ -0,0 +1 @@ +testExecutionConcurrency=-1 diff --git a/airbyte-integrations/connectors/destination-mssql-v2/icon.svg b/airbyte-integrations/connectors/destination-mssql-v2/icon.svg new file mode 100644 index 000000000000..edcaeb77c8f2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-mssql-v2/icon.svg @@ -0,0 +1 @@ +<svg xmlns="http://www.w3.org/2000/svg" width="250" height="250" fill="none"><path fill="url(#a)" d="m150.271 118.365-42.058 13.834-36.589 16.268-10.236 2.722a221.604 221.604 0 0 1-8.297 7.59c-3.242 2.818-6.262 5.381-8.583 7.238-2.575 2.049-6.39 5.892-8.329 8.326-2.893 3.65-5.181 7.525-6.167 10.504-1.748 5.38-.89 10.824 2.48 15.852 4.323 6.404 12.938 12.937 22.983 17.388 5.118 2.275 13.733 5.189 20.218 6.822 10.776 2.754 31.63 5.732 43.106 6.181 2.32.096 5.436.096 5.563 0 .254-.161 2.034-3.587 4.101-7.846 7.057-14.507 12.143-28.118 14.909-39.742 1.653-7.045 2.956-16.428 3.814-27.54.223-3.107.318-13.514.127-17.037-.286-5.764-.795-10.44-1.589-15.019-.127-.673-.159-1.281-.095-1.313.126-.096.508-.224 5.69-1.73l-1.049-2.497v-.001h.001Zm-9.601 5.668c.382 0 1.399 9.831 1.653 16.044.064 1.313.032 2.178-.032 2.178-.254 0-5.372-3.042-9.027-5.348-3.18-2.017-9.219-6.053-10.173-6.821-.318-.224-.287-.256 2.32-1.153 4.419-1.505 14.91-4.9 15.259-4.9Zm-21.426 7.11c.287 0 1.017.416 2.766 1.505 6.548 4.131 15.45 9.127 19.264 10.792 1.176.512 1.303.32-1.399 2.178-5.785 3.971-13.002 7.877-21.839 11.816-1.558.705-2.861 1.249-2.893 1.249-.063 0 .127-.8.381-1.761 2.13-7.975 3.338-16.044 3.402-22.513.032-3.203.032-3.203.318-3.298-.064.032-.032.032 0 .032Zm-4.418 1.697c.19.192.063 7.365-.191 9.319-.573 4.675-1.463 9.03-2.924 13.93-.35 1.185-.669 2.178-.731 2.242-.128.16-4.483-4.131-5.914-5.797-2.48-2.882-4.418-5.764-5.849-8.582-.731-1.441-1.875-4.259-1.78-4.355.508-.352 17.262-6.885 17.389-6.757Zm-20.79 8.23c.031 0 .063 0 .095.032.063.064.286.576.477 1.153 1.017 2.786 3.305 6.885 5.277 9.511 2.161 2.882 4.991 5.956 7.343 7.974.763.64 1.462 1.248 1.557 1.345.191.192.255.16-4.927 2.145-6.008 2.306-12.557 4.612-20.059 7.045-1.792.582-3.583 1.17-5.372 1.762-.286.096-.19-.064.636-1.377 3.72-5.861 9.377-17.357 12.556-25.491.541-1.409 1.081-2.818 1.177-3.138.127-.449.286-.609.699-.833.222-.064.445-.128.54-.128Zm-6.358 2.658c.095.064-1.526 3.49-3.116 6.629-3.083 6.052-6.453 12.009-10.967 19.246-.763 1.249-1.494 2.402-1.59 2.53-.158.225-.222.16-.73-.832-1.081-2.146-1.972-4.9-2.448-7.43-.477-2.498-.382-6.853.159-9.543.413-1.985.381-1.953 1.335-2.434 4.069-2.081 17.23-8.294 17.357-8.166Zm54.836 2.242v1.345c0 7.141-.763 16.94-1.876 24.082-.191 1.249-.35 2.273-.381 2.306 0 0-.922-.256-2.003-.577-4.768-1.505-9.95-3.715-14.623-6.308-3.084-1.698-7.566-4.484-7.439-4.612.032-.032 1.367-.737 2.925-1.569 6.231-3.267 12.207-6.789 17.388-10.28 1.94-1.313 4.864-3.426 5.5-4.003l.509-.384Zm-78.837 9.478c.127 0 .095.256-.096 1.409-.127.833-.286 2.37-.349 3.427-.255 4.675.509 8.134 2.797 12.874.636 1.313 1.145 2.401 1.113 2.433-.223.193-21.267 6.406-27.88 8.231-1.97.544-3.686 1.024-3.814 1.056-.222.064-.254.032-.159-.512.731-4.708 4.292-10.856 9.25-16.044 3.307-3.459 5.945-5.476 10.46-8.07 3.242-1.857 8.233-4.643 8.614-4.772 0-.032.032-.032.064-.032Zm49.591 8.935c.032-.032.794.385 1.716.929 6.803 3.971 16.276 7.654 24.35 9.543l.732.16-1.018.576c-4.227 2.37-18.119 8.198-32.329 13.547-2.066.768-4.1 1.537-4.482 1.697-.382.16-.732.256-.732.224s.573-1.153 1.304-2.53c3.973-7.493 7.979-16.62 10.013-22.961.255-.608.414-1.153.446-1.185Zm-5.055 1.665c.033.033-.222.705-.54 1.474-2.765 6.757-6.39 14.123-11.03 22.384-1.177 2.114-2.162 3.812-2.195 3.812-.031 0-.985-.577-2.13-1.282-6.739-4.163-12.715-9.287-16.625-14.25l-.572-.705 2.892-.801c10.364-2.849 19.169-5.924 27.911-9.702 1.24-.513 2.258-.93 2.289-.93Zm31.408 11.049s.032.032 0 0c.032.736-1.589 7.333-2.925 12.104-1.112 4.003-2.066 7.142-3.814 12.682-.763 2.434-1.43 4.451-1.463 4.451-.031 0-.222-.032-.413-.095-9.441-1.73-17.897-4.132-25.844-7.334-2.226-.897-5.405-2.338-5.595-2.498-.063-.065 1.844-.961 4.26-2.018 14.464-6.372 29.468-13.61 34.618-16.716.604-.384 1.081-.576 1.176-.576Zm-72.51 2.498c.063.064-3.974 5.924-9.633 13.898-1.97 2.786-4.26 6.052-5.118 7.269a128.074 128.074 0 0 0-2.893 4.292l-1.335 2.081-1.43-1.217c-1.686-1.409-4.61-4.419-5.913-6.084-2.735-3.426-4.578-7.045-5.31-10.343-.349-1.538-.349-2.307-.031-2.402.476-.128 8.964-2.146 16.912-4.004 4.419-1.024 9.536-2.241 11.38-2.689 1.844-.449 3.339-.801 3.37-.801Zm4.068 1.569 1.017 1.153c4.578 5.156 9.251 8.967 14.91 12.297 1.017.577 1.78 1.089 1.716 1.121-.222.16-19.646 7.109-28.642 10.247-5.054 1.794-9.219 3.235-9.25 3.235-.032 0-.318-.192-.637-.417l-.571-.416.921-1.345c2.988-4.355 6.74-9.127 14.91-19.054l5.626-6.821Zm25.368 18.285c.031-.031 1.43.481 3.147 1.122 4.133 1.569 7.406 2.561 11.794 3.682 5.404 1.377 13.224 2.722 17.834 3.107.699.063 1.08.128.953.224-.223.128-4.895 1.697-8.329 2.786-5.468 1.729-22.157 6.693-35.762 10.632a371.276 371.276 0 0 1-4.8 1.377c-.318.064-1.368-.225-1.368-.353 0-.063.763-1.056 1.685-2.145 4.578-5.508 9.124-11.657 12.907-17.485 1.049-1.602 1.94-2.915 1.94-2.947Zm-5.595.161c.031.032-2.225 3.682-6.167 9.959-1.685 2.658-3.56 5.668-4.228 6.725-.636 1.025-1.59 2.626-2.13 3.523l-.922 1.633-.477-.128c-1.144-.32-9.187-3.171-11.317-4.035-2.639-1.057-5.372-2.338-7.407-3.427-2.543-1.377-5.721-3.427-5.467-3.49.063-.032 4.418-1.218 9.663-2.658 13.924-3.812 21.649-6.021 26.704-7.622.922-.288 1.716-.513 1.748-.48Zm39.577 9.35h.032c.127.321-5.022 14.732-6.899 19.279-.412 1.024-.571 1.281-.794 1.248-.54-.031-8.01-1.088-12.557-1.761-7.915-1.216-21.203-3.555-24.54-4.323l-.763-.16 4.736-1.089c10.172-2.306 15.068-3.555 20.027-5.092 6.262-1.921 12.462-4.323 18.724-7.269.985-.448 1.812-.801 2.034-.833Z"/><path fill="url(#b)" d="M106.907 19.603c-.699-.096-11.984 4.003-19.264 6.98-9.823 4.036-17.452 7.88-22.157 11.21-1.748 1.249-3.942 3.458-4.291 4.322-.127.32-.191.705-.191 1.09l4.26 4.066 10.14 3.267 24.128 4.355 27.593 4.772.286-2.402c-.095 0-.159-.032-.253-.032l-3.625-.577-.731-1.312c-3.751-6.661-7.884-14.924-10.3-20.496-1.875-4.323-3.655-9.318-4.641-12.905-.54-2.178-.604-2.306-.953-2.338h-.001Zm-.509 1.633h.033c.031.032.158.93.286 1.985.54 4.484 1.526 8.807 3.083 13.483 1.176 3.522 1.176 3.33-.19 2.914-3.243-.897-17.77-3.427-28.293-4.9-1.684-.224-3.115-.448-3.115-.48-.127-.128 7.597-4.195 10.999-5.796 4.355-2.018 16.307-7.046 17.197-7.206ZM75.754 35.583l1.24.416c6.739 2.306 23.682 5.572 33.028 6.34 1.049.097 1.94.193 1.971.193.032.032-.858.513-2.003 1.057-4.513 2.273-9.473 5.06-12.906 7.205-1.017.64-1.94 1.153-2.066 1.153-.127 0-.794-.128-1.494-.224l-1.271-.192-3.179-3.139a1335.107 1335.107 0 0 0-11.667-11.304l-1.653-1.505Zm-1.272.992 4.483 5.637c2.447 3.106 4.927 6.148 5.468 6.82.54.673.985 1.218.953 1.25-.127.096-6.484-1.153-9.854-1.921-3.465-.801-4.896-1.186-7.026-1.858l-1.748-.576v-.45c.031-2.145 2.734-5.347 7.311-8.613l.413-.289Zm38.084 7.686c.127 0 .286.288.667 1.153 1.081 2.402 4.451 8.87 5.277 10.12.255.416.699.448-3.783-.288-10.776-1.762-14.241-2.338-14.241-2.402 0-.032.318-.256.731-.48 3.338-1.858 6.707-4.228 9.696-6.758.731-.608 1.399-1.184 1.525-1.28.032-.065.096-.097.128-.065Z"/><path fill="url(#c)" d="M61.42 41.667s-.698 1.121-.03 2.787c.413 1.024 1.62 2.273 2.987 3.554 0 0 14.147 13.898 15.863 15.884 7.82 9.095 11.222 18.061 11.54 30.423.19 7.942-1.304 14.923-5.023 23.025-6.612 14.539-20.568 30.583-42.089 48.388l3.148-1.057c2.034-1.537 4.8-3.17 11.284-6.757 14.973-8.262 31.821-15.852 52.484-23.666 29.754-11.272 78.678-24.466 106.525-28.757l2.893-.449-.445-.704c-2.543-3.971-4.292-6.437-6.39-9.063-6.103-7.622-13.51-13.802-22.57-18.926-12.461-7.013-28.579-12.489-48.987-16.556-3.846-.769-12.302-2.242-19.169-3.299-14.559-2.273-23.969-3.842-34.332-5.636-3.719-.64-9.282-1.6-12.97-2.401-1.907-.417-5.563-1.282-8.424-2.274-2.288-.897-5.595-1.794-6.294-4.516Zm8.203 8.006c.031-.031.54.16 1.208.384a82.01 82.01 0 0 0 4.61 1.41c1.395.386 2.793.76 4.196 1.12 1.907.48 3.496.929 3.528.929.223.224 3.433 10.567 4.514 14.538.413 1.506.731 2.787.7 2.787-.033.032-.382-.513-.795-1.25-3.72-6.596-9.6-13.29-16.404-18.67-.89-.64-1.557-1.216-1.557-1.248Zm15.64 4.355c.16 0 .858.097 1.716.289 5.405 1.216 15.1 3.074 21.299 4.13 1.049.16 1.876.353 1.876.417s-.382.288-.859.545c-1.048.544-5.277 3.073-6.675 4.034-3.529 2.37-6.708 4.932-8.996 7.238-.922.928-1.717 1.697-1.717 1.697s-.19-.544-.35-1.217c-1.144-4.451-3.529-11.049-5.69-15.692-.35-.736-.636-1.409-.636-1.473 0 .064 0 .032.032.032Zm27.529 5.284c.191.064.509 1.153 1.144 3.555 1.177 4.644 1.717 9.831 1.527 14.667-.064 1.345-.128 2.594-.192 2.754l-.095.32-1.653-.544c-3.401-1.089-8.932-2.722-13.668-4.067-2.703-.737-4.896-1.41-4.896-1.473 0-.193 3.942-4.163 5.627-5.668 3.21-2.851 11.92-9.64 12.206-9.544Zm2.194.32c.095-.095 13.16 2.178 19.105 3.331 4.419.864 10.84 2.21 11.221 2.37.191.064-.477.448-2.607 1.409-8.392 3.81-14.623 7.237-20.821 11.4-1.621 1.09-2.988 1.986-3.02 1.986-.032 0-.063-.929-.063-2.05 0-6.084-1.208-12.233-3.434-17.42-.222-.513-.413-.993-.381-1.026Zm33.791 6.726c.096.096-.318 2.69-.699 4.227-1.144 4.771-4.228 11.85-8.01 18.51-.668 1.185-1.272 2.145-1.336 2.177-.063.032-.922-.448-1.907-1.025-3.688-2.177-7.884-4.227-12.462-6.148-1.271-.545-2.384-.993-2.415-1.057-.223-.192 10.013-7.013 15.417-10.28 4.292-2.626 11.285-6.533 11.412-6.404Zm2.416.384c.286 0 6.072 1.601 9.092 2.498 7.47 2.242 16.053 5.412 21.648 7.974l2.32 1.057-1.62.384c-13.669 3.17-25.368 6.821-36.653 11.433-.922.384-1.749.704-1.812.704-.064 0 .254-.736.668-1.633 3.401-7.27 5.595-14.86 6.134-21.329.033-.608.128-1.088.223-1.088ZM93.592 80.064c.095-.096 4.514.96 6.898 1.633 3.624 1.025 11.316 3.618 11.316 3.81 0 .033-.858.77-1.874 1.666-4.165 3.49-8.171 7.173-12.971 11.849-1.43 1.377-2.638 2.498-2.701 2.498-.064 0-.096-.193-.064-.449.73-5.38.572-12.297-.445-19.31-.096-.897-.191-1.665-.16-1.697Zm92.856.096c.062.064-2.035 3.394-3.37 5.284-1.907 2.754-4.706 6.405-11.032 14.41a2153.329 2153.329 0 0 0-8.328 10.601c-1.272 1.601-2.32 2.945-2.353 2.945-.031 0-.444-.576-.889-1.28-3.561-5.38-7.821-10.088-12.875-14.315-.954-.8-2.002-1.665-2.352-1.922-.35-.256-.636-.512-.636-.544 0-.096 5.403-2.434 9.505-4.099 7.184-2.946 16.974-6.469 24.318-8.742 3.847-1.217 7.948-2.402 8.012-2.338Zm2.447.64c.127-.031.89.353 1.812.897 7.725 4.451 15.29 10.183 21.267 16.076 1.685 1.666 5.849 6.021 5.785 6.053 0 0-1.462.128-3.178.256-13.384 1.025-30.517 3.874-46.984 7.877-1.113.256-2.098.48-2.162.48-.063 0 1.176-1.248 2.734-2.753 9.664-9.383 14.083-15.308 19.296-25.876.731-1.569 1.367-2.913 1.43-3.01-.032 0-.032 0 0 0Zm-70.794 7.302c.445.096 4.578 2.05 7.693 3.619 2.861 1.44 7.153 3.746 7.375 3.938.032.032-1.494.833-3.369 1.762-5.977 3.01-11.095 5.86-16.436 9.126-1.525.929-2.797 1.698-2.829 1.698-.127 0-.095-.129.763-1.698 2.861-5.251 5.15-11.528 6.454-17.645.126-.48.254-.8.349-.8Zm-4.133.768c.096.097-.985 4.035-1.652 6.181-1.304 4.067-3.497 9.159-5.627 13.002-.509.896-1.272 2.209-1.685 2.945l-.795 1.281-1.78-1.728c-2.066-2.018-3.751-3.267-5.912-4.388-.86-.448-1.527-.833-1.527-.896 0-.257 5.436-5.22 9.601-8.807 2.988-2.594 9.282-7.686 9.377-7.59Zm25.273 10.472 1.558 1.025c3.559 2.338 7.755 5.444 10.967 8.166 1.811 1.505 5.308 4.676 6.008 5.444l.381.417-2.575.736c-14.559 4.067-25.812 7.686-38.941 12.553-1.463.545-2.703.993-2.798.993-.19 0-.35.16 2.925-2.882 8.392-7.782 15.831-16.365 21.362-24.723l1.113-1.729Zm-6.644 1.666c.063.063-4.292 6.244-6.899 9.735-3.115 4.163-8.646 11.144-12.461 15.691-1.589 1.89-2.956 3.459-3.02 3.491-.095.032-.127-.449-.127-1.185 0-3.875-.985-8.006-2.702-11.529-.731-1.473-.858-1.825-.699-1.985.604-.545 9.854-5.861 15.703-9.031 3.942-2.113 10.109-5.252 10.205-5.187Zm-40.182 9.927c.096 0 .827.384 1.654.833a34.866 34.866 0 0 1 5.467 3.714c.064.064-.763.737-1.844 1.537a298.67 298.67 0 0 0-10.267 7.814c-2.798 2.242-2.893 2.306-2.575 1.826 2.098-3.235 3.147-5.06 4.26-7.398a59.83 59.83 0 0 0 2.67-6.693c.254-.928.572-1.633.635-1.633Zm10.714 8.454c.158-.032.349.256 1.207 1.537 1.812 2.722 3.211 6.373 3.561 9.319l.063.641-4.354 1.697c-7.79 3.042-14.973 6.053-19.837 8.294a349.359 349.359 0 0 0-5.309 2.563c-1.557.8-2.829 1.408-2.829 1.376 0-.032.985-.768 2.193-1.665 9.506-6.949 17.739-14.571 23.906-22.193.667-.8 1.272-1.537 1.335-1.569h.064Zm-4.928 1.217c.128.128-3.496 4.259-5.976 6.789-6.135 6.309-12.207 11.241-19.741 16.045-.954.608-1.812 1.152-1.907 1.216-.223.128.063-.192 3.37-3.811 2.097-2.274 3.687-4.195 5.499-6.564 1.207-1.569 1.43-1.794 3.179-3.043 4.673-3.395 15.45-10.76 15.576-10.632Z"/><defs><linearGradient id="a" x1="60.77" x2="76.036" y1="210.234" y2="201.28" gradientUnits="userSpaceOnUse"><stop stop-color="#909CA9"/><stop offset="1" stop-color="#EDEDEE"/></linearGradient><linearGradient id="b" x1="61.115" x2="73.352" y1="39.571" y2="39.571" gradientUnits="userSpaceOnUse"><stop stop-color="#939FAB"/><stop offset="1" stop-color="#DCDEE1"/></linearGradient><radialGradient id="c" cx="0" cy="0" r="1" gradientTransform="rotate(-171.395 65.048 43.431) scale(15.7789 31.7884)" gradientUnits="userSpaceOnUse"><stop stop-color="#EE352C"/><stop offset="1" stop-color="#A91D22"/></radialGradient></defs></svg> \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-mssql-v2/metadata.yaml b/airbyte-integrations/connectors/destination-mssql-v2/metadata.yaml new file mode 100644 index 000000000000..1b09be9a1ad8 --- /dev/null +++ b/airbyte-integrations/connectors/destination-mssql-v2/metadata.yaml @@ -0,0 +1,34 @@ +data: + connectorSubtype: database + connectorType: destination + definitionId: 37a928c1-2d5c-431a-a97d-ae236bd1ea0c + dockerImageTag: 0.1.0 + dockerRepository: airbyte/destination-mssql-v2 + githubIssueLabel: destination-mssql-v2 + icon: icon.svg + license: ELv2 + name: MSSQL V2 Destination + registryOverrides: + cloud: + enabled: false + oss: + enabled: false + releaseStage: alpha + documentationUrl: https://docs.airbyte.com/integrations/destinations/mssql-v2 + tags: + - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community + supportsRefreshes: true + connectorTestSuitesOptions: + - suite: unitTests + - suite: integrationTests + testSecrets: + - name: SECRET_DESTINATION-S3-V2-MINIMAL-REQUIRED-CONFIG + fileName: s3_dest_v2_minimal_required_config.json + secretStore: + type: GSM + alias: airbyte-connector-testing-secret-store +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-mssql-v2/src/main/kotlin/io/airbyte/integrations/destination/mssql/v2/MSSQLDestination.kt b/airbyte-integrations/connectors/destination-mssql-v2/src/main/kotlin/io/airbyte/integrations/destination/mssql/v2/MSSQLDestination.kt new file mode 100644 index 000000000000..cb1d288390aa --- /dev/null +++ b/airbyte-integrations/connectors/destination-mssql-v2/src/main/kotlin/io/airbyte/integrations/destination/mssql/v2/MSSQLDestination.kt @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.mssql.v2 + +import io.airbyte.cdk.AirbyteDestinationRunner + +object MSSQLDestination { + @JvmStatic + fun main(args: Array<String>) { + AirbyteDestinationRunner.run(*args) + } +} diff --git a/airbyte-integrations/connectors/destination-mssql-v2/src/main/kotlin/io/airbyte/integrations/destination/mssql/v2/config/MSSQLConfiguration.kt b/airbyte-integrations/connectors/destination-mssql-v2/src/main/kotlin/io/airbyte/integrations/destination/mssql/v2/config/MSSQLConfiguration.kt new file mode 100644 index 000000000000..ca468c08caa7 --- /dev/null +++ b/airbyte-integrations/connectors/destination-mssql-v2/src/main/kotlin/io/airbyte/integrations/destination/mssql/v2/config/MSSQLConfiguration.kt @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.mssql.v2.config + +import dagger.Component.Factory +import io.airbyte.cdk.load.command.DestinationConfiguration +import io.airbyte.cdk.load.command.DestinationConfigurationFactory +import jakarta.inject.Singleton + +data class MSSQLConfiguration(val placeholder: String) : DestinationConfiguration() + +@Singleton +class MSSQLConfigurationFactory : + DestinationConfigurationFactory<MSSQLSpecification, MSSQLConfiguration> { + override fun makeWithoutExceptionHandling(pojo: MSSQLSpecification): MSSQLConfiguration { + TODO("Not yet implemented") + } +} + +@Factory +class MSSQLConfigurationProvider(private val config: DestinationConfiguration) { + @Singleton + fun get(): MSSQLConfiguration { + return config as MSSQLConfiguration + } +} diff --git a/airbyte-integrations/connectors/destination-mssql-v2/src/main/kotlin/io/airbyte/integrations/destination/mssql/v2/config/MSSQLSpecification.kt b/airbyte-integrations/connectors/destination-mssql-v2/src/main/kotlin/io/airbyte/integrations/destination/mssql/v2/config/MSSQLSpecification.kt new file mode 100644 index 000000000000..30a7f9769c63 --- /dev/null +++ b/airbyte-integrations/connectors/destination-mssql-v2/src/main/kotlin/io/airbyte/integrations/destination/mssql/v2/config/MSSQLSpecification.kt @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.mssql.v2.config + +import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle +import io.airbyte.cdk.command.ConfigurationSpecification +import io.airbyte.cdk.load.spec.DestinationSpecificationExtension +import io.airbyte.protocol.models.v0.DestinationSyncMode +import jakarta.inject.Singleton + +@Singleton +@JsonSchemaTitle("MSSQL V2 Destination Specification") +class MSSQLSpecification : ConfigurationSpecification() {} + +@Singleton +class MSSQLSpecificationExtension : DestinationSpecificationExtension { + override val supportedSyncModes = + listOf( + DestinationSyncMode.OVERWRITE, + DestinationSyncMode.APPEND, + DestinationSyncMode.APPEND_DEDUP + ) + override val supportsIncremental = true +} diff --git a/docs/integrations/destinations/mssql-v2.md b/docs/integrations/destinations/mssql-v2.md new file mode 100644 index 000000000000..0e168a4ea5b7 --- /dev/null +++ b/docs/integrations/destinations/mssql-v2.md @@ -0,0 +1,12 @@ +# MSSQL (V2) + +## Changelog + +<details> + <summary>Expand to review</summary> + +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :--------------------------------------------------------- |:---------------| +| 0.1.0 | 2024-12-16 | [\#49460](https://github.com/airbytehq/airbyte/pull/49460) | Initial commit | + +</details> \ No newline at end of file From b7e25c99eb550f50997c1f3c46473df0d556093a Mon Sep 17 00:00:00 2001 From: Christo Grabowski <108154848+ChristoGrab@users.noreply.github.com> Date: Tue, 17 Dec 2024 08:21:01 -0800 Subject: [PATCH 3/7] feat(source-pardot): 23 new streams, fixed auth flow (#49424) Co-authored-by: Justin Beasley <justin.m.beasley@gmail.com> Co-authored-by: Danylo Jablonski <150933663+DanyloGL@users.noreply.github.com> --- .../connectors/source-pardot/manifest.yaml | 6021 +++++++++++++---- .../connectors/source-pardot/metadata.yaml | 9 +- .../integrations/sources/pardot-migrations.md | 14 + docs/integrations/sources/pardot.md | 117 +- 4 files changed, 4892 insertions(+), 1269 deletions(-) create mode 100644 docs/integrations/sources/pardot-migrations.md diff --git a/airbyte-integrations/connectors/source-pardot/manifest.yaml b/airbyte-integrations/connectors/source-pardot/manifest.yaml index 495ec71883a6..13ac303ce829 100644 --- a/airbyte-integrations/connectors/source-pardot/manifest.yaml +++ b/airbyte-integrations/connectors/source-pardot/manifest.yaml @@ -1,1226 +1,4795 @@ -version: 5.13.0 - -type: DeclarativeSource - -check: - type: CheckStream - stream_names: - - campaigns - -definitions: - streams: - campaigns: - type: DeclarativeStream - name: campaigns - primary_key: - - id - retriever: - type: SimpleRetriever - requester: - $ref: "#/definitions/base_requester" - path: campaign/version/4/do/query - http_method: GET - request_parameters: - format: json - sort_by: id - sort_order: ascending - created_after: "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%SZ') }}" - request_headers: - Pardot-Business-Unit-Id: "{{ config[\"pardot_business_unit_id\"] }}" - record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - result - - campaign - paginator: - type: DefaultPaginator - page_token_option: - type: RequestOption - inject_into: request_parameter - field_name: id_greater_than - pagination_strategy: - type: CursorPagination - cursor_value: "{{ response.get(\"campaign\", {})[-1].get(\"id\", {}) }}" - stop_condition: "{{ not response.get(\"campaign\", {})[-1].get(\"id\", {}) }}" - schema_loader: - type: InlineSchemaLoader - schema: - $ref: "#/schemas/campaigns" - email_clicks: - type: DeclarativeStream - name: email_clicks - primary_key: - - id - retriever: - type: SimpleRetriever - requester: - $ref: "#/definitions/base_requester" - path: emailClick/version/4/do/query - http_method: GET - request_parameters: - format: json - created_after: "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%SZ') }}" - request_headers: - Pardot-Business-Unit-Id: "{{ config[\"pardot_business_unit_id\"] }}" - record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - result - - emailClick - paginator: - type: DefaultPaginator - page_token_option: - type: RequestOption - inject_into: request_parameter - field_name: id_greater_than - pagination_strategy: - type: CursorPagination - cursor_value: "{{ response.get(\"emailClick\", {})[-1].get(\"id\", {}) }}" - stop_condition: "{{ not response.get(\"emailClick\", {})[-1].get(\"id\", {}) }}" - schema_loader: - type: InlineSchemaLoader - schema: - $ref: "#/schemas/email_clicks" - list_membership: - type: DeclarativeStream - name: list_membership - primary_key: - - id - retriever: - type: SimpleRetriever - requester: - $ref: "#/definitions/base_requester" - path: listMembership/version/4/do/query - http_method: GET - request_parameters: - format: json - sort_by: id - sort_order: ascending - created_after: "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%SZ') }}" - request_headers: - Pardot-Business-Unit-Id: "{{ config[\"pardot_business_unit_id\"] }}" - record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - result - - list_membership - paginator: - type: DefaultPaginator - page_token_option: - type: RequestOption - inject_into: request_parameter - field_name: updated_after - pagination_strategy: - type: CursorPagination - cursor_value: >- - {{ response.get("list_membership", {})[-1].get("updated_at", {}) - }} - stop_condition: >- - {{ not response.get("list_membership", {})[-1].get("updated_at", - {}) }} - schema_loader: - type: InlineSchemaLoader - schema: - $ref: "#/schemas/list_membership" - lists: - type: DeclarativeStream - name: lists - primary_key: - - id - retriever: - type: SimpleRetriever - requester: - $ref: "#/definitions/base_requester" - path: list/version/4/do/query - http_method: GET - request_parameters: - format: json - sort_by: updated_at - sort_order: ascending - created_after: "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%SZ') }}" - request_headers: - Pardot-Business-Unit-Id: "{{ config[\"pardot_business_unit_id\"] }}" - record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - result - - list - paginator: - type: DefaultPaginator - page_token_option: - type: RequestOption - inject_into: request_parameter - field_name: updated_after - pagination_strategy: - type: CursorPagination - cursor_value: "{{ response.get(\"list\", {})[-1].get(\"updated_at\", {}) }}" - stop_condition: "{{ not response.get(\"list\", {})[-1].get(\"updated_at\", {}) }}" - schema_loader: - type: InlineSchemaLoader - schema: - $ref: "#/schemas/lists" - prospect_accounts: - type: DeclarativeStream - name: prospect_accounts - primary_key: - - id - retriever: - type: SimpleRetriever - requester: - $ref: "#/definitions/base_requester" - path: prospectAccount/version/4/do/query - http_method: GET - request_parameters: - format: json - sort_by: updated_at - sort_order: ascending - created_after: "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%SZ') }}" - request_headers: - Pardot-Business-Unit-Id: "{{ config[\"pardot_business_unit_id\"] }}" - record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - result - - prospectAccount - paginator: - type: DefaultPaginator - page_token_option: - type: RequestOption - inject_into: request_parameter - field_name: updated_after - pagination_strategy: - type: CursorPagination - cursor_value: >- - {{ response.get("prospectAccount", {})[-1].get("updated_at", {}) - }} - stop_condition: >- - {{ not response.get("prospectAccount", {})[-1].get("updated_at", - {}) }} - schema_loader: - type: InlineSchemaLoader - schema: - $ref: "#/schemas/prospect_accounts" - prospects: - type: DeclarativeStream - name: prospects - primary_key: - - id - retriever: - type: SimpleRetriever - requester: - $ref: "#/definitions/base_requester" - path: prospect/version/4/do/query - http_method: GET - request_parameters: - format: json - sort_by: updated_at - sort_order: ascending - created_after: "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%SZ') }}" - request_headers: - Pardot-Business-Unit-Id: "{{ config[\"pardot_business_unit_id\"] }}" - record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - result - - prospect - paginator: - type: DefaultPaginator - page_token_option: - type: RequestOption - inject_into: request_parameter - field_name: updated_after - pagination_strategy: - type: CursorPagination - cursor_value: "{{ response.get(\"prospect\", {})[-1].get(\"updated_at\", {}) }}" - stop_condition: "{{ not response.get(\"prospect\", {})[-1].get(\"updated_at\", {}) }}" - schema_loader: - type: InlineSchemaLoader - schema: - $ref: "#/schemas/prospects" - users: - type: DeclarativeStream - name: users - primary_key: - - id - retriever: - type: SimpleRetriever - requester: - $ref: "#/definitions/base_requester" - path: user - http_method: GET - request_parameters: - format: json - created_after: "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%SZ') }}" - request_headers: - Pardot-Business-Unit-Id: "{{ config[\"pardot_business_unit_id\"] }}" - record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - result - - user - paginator: - type: DefaultPaginator - page_token_option: - type: RequestOption - inject_into: request_parameter - field_name: created_after - pagination_strategy: - type: CursorPagination - cursor_value: "{{ response.get(\"user\", {})[-1].get(\"created_at\", {}) }}" - stop_condition: "{{ not response.get(\"user\", {})[-1].get(\"created_at\", {}) }}" - schema_loader: - type: InlineSchemaLoader - schema: - $ref: "#/schemas/users" - visitor_activities: - type: DeclarativeStream - name: visitor_activities - primary_key: - - id - retriever: - type: SimpleRetriever - requester: - $ref: "#/definitions/base_requester" - path: visitorActivity/version/4/do/query - http_method: GET - request_parameters: - format: json - created_after: "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%SZ') }}" - request_headers: - Pardot-Business-Unit-Id: "{{ config[\"pardot_business_unit_id\"] }}" - record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - result - - visitor_activity - paginator: - type: DefaultPaginator - page_token_option: - type: RequestOption - inject_into: request_parameter - field_name: id_greater_than - pagination_strategy: - type: CursorPagination - cursor_value: "{{ response.get(\"visitor_activity\", {})[-1].get(\"id\", {}) }}" - stop_condition: "{{ not response.get(\"visitor_activity\", {})[-1].get(\"id\", {}) }}" - schema_loader: - type: InlineSchemaLoader - schema: - $ref: "#/schemas/visitor_activities" - visitors: - type: DeclarativeStream - name: visitors - primary_key: - - id - retriever: - type: SimpleRetriever - requester: - $ref: "#/definitions/base_requester" - path: visitor/version/4/do/query - http_method: GET - request_parameters: - format: json - sort_by: updated_at - sort_order: ascending - created_after: "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%SZ') }}" - request_headers: - Pardot-Business-Unit-Id: "{{ config[\"pardot_business_unit_id\"] }}" - record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - result - - visitor - paginator: - type: DefaultPaginator - page_token_option: - type: RequestOption - inject_into: request_parameter - field_name: updated_after - pagination_strategy: - type: CursorPagination - cursor_value: "{{ response.get(\"visitor\", {})[-1].get(\"updated_at\", {}) }}" - stop_condition: "{{ not response.get(\"visitor\", {})[-1].get(\"updated_at\", {}) }}" - schema_loader: - type: InlineSchemaLoader - schema: - $ref: "#/schemas/visitors" - visits: - type: DeclarativeStream - name: visits - retriever: - type: SimpleRetriever - requester: - $ref: "#/definitions/base_requester" - path: visit/version/4/do/query - http_method: GET - request_parameters: - format: json - sort_order: ascending - created_after: "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%SZ') }}" - request_headers: - Pardot-Business-Unit-Id: "{{ config[\"pardot_business_unit_id\"] }}" - record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - result - - visit - schema_loader: - type: InlineSchemaLoader - schema: - $ref: "#/schemas/visits" - opportunities: - type: DeclarativeStream - name: opportunities - primary_key: - - id - retriever: - type: SimpleRetriever - requester: - $ref: "#/definitions/base_requester" - path: opportunity/version/4/do/query - http_method: GET - request_parameters: - format: json - created_after: "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%SZ') }}" - request_headers: - Pardot-Business-Unit-Id: "{{ config[\"pardot_business_unit_id\"] }}" - record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - result - - opportunity - paginator: - type: DefaultPaginator - page_token_option: - type: RequestOption - inject_into: request_parameter - field_name: created_after - pagination_strategy: - type: CursorPagination - cursor_value: "{{ response.get(\"opportunity\", {})[-1].get(\"created_at\", {}) }}" - stop_condition: >- - {{ not response.get("opportunity", {})[-1].get("created_at", {}) - }} - schema_loader: - type: InlineSchemaLoader - schema: - $ref: "#/schemas/opportunities" - base_requester: - type: HttpRequester - url_base: https://pi.pardot.com/api/ - authenticator: - type: OAuthAuthenticator - client_id: "{{ config[\"client_id\"] }}" - grant_type: refresh_token - client_secret: "{{ config[\"client_secret\"] }}" - refresh_token: "{{ config[\"client_refresh_token\"] }}" - refresh_request_body: {} - token_refresh_endpoint: >- - https://{{ 'test' if config['is_sandbox'] else 'login' - }}.salesforce.com/services/oauth2/token - -streams: - - $ref: "#/definitions/streams/campaigns" - - $ref: "#/definitions/streams/email_clicks" - - $ref: "#/definitions/streams/list_membership" - - $ref: "#/definitions/streams/lists" - - $ref: "#/definitions/streams/prospect_accounts" - - $ref: "#/definitions/streams/prospects" - - $ref: "#/definitions/streams/users" - - $ref: "#/definitions/streams/visitor_activities" - - $ref: "#/definitions/streams/visitors" - - $ref: "#/definitions/streams/visits" - # - $ref: "#/definitions/streams/opportunities" # Currently disabled because test account doesn't have any data - - -spec: - type: Spec - connection_specification: - type: object - $schema: http://json-schema.org/draft-07/schema# - required: - - client_id - - client_secret - - refresh_token - - pardot_business_unit_id - properties: - client_id: - type: string - description: The Consumer Key that can be found when viewing your app in Salesforce - airbyte_secret: true - order: 0 - is_sandbox: - type: boolean - description: >- - Whether or not the the app is in a Salesforce sandbox. If you do not - know what this, assume it is false. - default: false - order: 1 - start_date: - type: string - description: >- - UTC date and time in the format 2017-01-25T00:00:00Z. Any data before - this date will not be replicated. Leave blank to skip this filter - default: null - pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ - examples: - - "2021-07-25T00:00:00Z" - order: 2 - client_secret: - type: string - description: >- - The Consumer Secret that can be found when viewing your app in - Salesforce - airbyte_secret: true - order: 3 - refresh_token: - type: string - description: >- - Salesforce Refresh Token used for Airbyte to access your Salesforce - account. If you don't know what this is, follow this <a - href="https://medium.com/@bpmmendis94/obtain-access-refresh-tokens-from-salesforce-rest-api-a324fe4ccd9b">guide</a> - to retrieve it. - airbyte_secret: true - order: 4 - pardot_business_unit_id: - type: string - description: >- - Pardot Business ID, can be found at Setup > Pardot > Pardot Account - Setup - order: 5 - additionalProperties: true - -metadata: - autoImportSchema: - campaigns: false - email_clicks: false - list_membership: false - lists: false - prospect_accounts: false - prospects: false - users: false - visitor_activities: false - visitors: false - visits: false - opportunities: false - yamlComponents: - global: - - authenticator - testedStreams: - campaigns: - streamHash: ef6fbf0ab7ffa8f81749103d94235abd001d0f39 - email_clicks: - streamHash: 2d74c8aab6c19a77ed2315e36c2fcb18296b641a - list_membership: - streamHash: 33edc450360e922bd939bf7aea5bb756e8714d64 - lists: - streamHash: 984611ea793da473a3cfa6968fb01a4a36b6145f - prospect_accounts: - streamHash: 5938979e371a63739b281c64cd4f61064331d513 - prospects: - streamHash: 4bcf3735358cd3942c17750ef76e052faf5a034e - users: - streamHash: 21012c537f20aed85d8fca2a91c135a2ed9a8e34 - visitor_activities: - streamHash: 857eb94e1cf16eac22e0386450e79abfeffdf347 - visitors: - streamHash: 339bee895c4c803eccc98ba0b836fbc0cb36d937 - visits: - streamHash: c526ae3538c2c4f9b434e803521c509c0e5271a2 - opportunities: - streamHash: fe2484cadf31a482f5f5421ac4a70d14f03e8796 - assist: {} - -schemas: - campaigns: - type: object - $schema: http://json-schema.org/draft-07/schema# - additionalProperties: true - properties: - cost: - type: - - "null" - - integer - id: - type: - - integer - name: - type: - - "null" - - string - email_clicks: - type: object - $schema: http://json-schema.org/draft-07/schema# - additionalProperties: true - properties: - created_at: - type: - - "null" - - string - format: date-time - drip_program_action_id: - type: - - "null" - - integer - email_template_id: - type: - - "null" - - integer - id: - type: - - integer - list_email_id: - type: - - "null" - - integer - prospect_id: - type: - - "null" - - integer - tracker_redirect_id: - type: - - "null" - - integer - url: - type: - - "null" - - string - list_membership: - type: object - $schema: http://json-schema.org/draft-07/schema# - additionalProperties: true - properties: - created_at: - type: - - "null" - - string - format: date-time - id: - type: - - integer - list_id: - type: - - integer - opted_out: - type: - - "null" - - boolean - prospect_id: - type: - - integer - updated_at: - type: - - "null" - - string - format: date-time - lists: - type: object - $schema: http://json-schema.org/draft-07/schema# - additionalProperties: true - properties: - description: - type: - - "null" - - string - created_at: - type: - - "null" - - string - format: date-time - id: - type: - - integer - is_crm_visible: - type: - - "null" - - boolean - is_dynamic: - type: - - "null" - - boolean - is_public: - type: - - "null" - - boolean - name: - type: - - "null" - - string - title: - type: - - "null" - - string - updated_at: - type: - - "null" - - string - format: date-time - prospect_accounts: - type: object - $schema: http://json-schema.org/draft-07/schema# - additionalProperties: true - properties: - assigned_to: - type: - - "null" - - object - created_at: - type: - - "null" - - string - format: date-time - id: - type: - - integer - name: - type: - - "null" - - string - updated_at: - type: - - "null" - - string - format: date-time - prospects: - type: object - $schema: http://json-schema.org/draft-07/schema# - additionalProperties: true - properties: - address_one: - type: - - "null" - - string - address_two: - type: - - "null" - - string - annual_revenue: - type: - - "null" - - string - campaign_id: - type: - - "null" - - integer - city: - type: - - "null" - - string - comments: - type: - - "null" - - string - company: - type: - - "null" - - string - country: - type: - - "null" - - string - created_at: - type: - - "null" - - string - format: date-time - crm_account_fid: - type: - - "null" - - string - crm_contact_fid: - type: - - "null" - - string - crm_last_sync: - type: - - "null" - - string - format: date-time - crm_lead_fid: - type: - - "null" - - string - crm_owner_fid: - type: - - "null" - - string - crm_url: - type: - - "null" - - string - department: - type: - - "null" - - string - email: - type: - - "null" - - string - employees: - type: - - "null" - - string - fax: - type: - - "null" - - string - first_name: - type: - - "null" - - string - grade: - type: - - "null" - - string - id: - type: - - integer - industry: - type: - - "null" - - string - is_do_not_call: - type: - - "null" - - integer - is_do_not_email: - type: - - "null" - - integer - is_reviewed: - type: - - "null" - - integer - is_starred: - type: - - "null" - - integer - job_title: - type: - - "null" - - string - last_activity_at: - type: - - "null" - - string - format: date-time - last_name: - type: - - "null" - - string - notes: - type: - - "null" - - string - opted_out: - type: - - "null" - - integer - password: - type: - - "null" - - string - phone: - type: - - "null" - - string - prospect_account_id: - type: - - "null" - - integer - recent_interaction: - type: - - "null" - - string - salutation: - type: - - "null" - - string - score: - type: - - "null" - - integer - source: - type: - - "null" - - string - state: - type: - - "null" - - string - territory: - type: - - "null" - - string - updated_at: - type: - - "null" - - string - format: date-time - website: - type: - - "null" - - string - years_in_business: - type: - - "null" - - string - zip: - type: - - "null" - - string - users: - type: object - $schema: http://json-schema.org/draft-07/schema# - additionalProperties: true - properties: - created_at: - type: - - "null" - - string - format: date-time - email: - type: - - "null" - - string - first_name: - type: - - "null" - - string - id: - type: - - integer - job_title: - type: - - "null" - - string - last_name: - type: - - "null" - - string - role: - type: - - "null" - - string - updated_at: - type: - - "null" - - string - format: date-time - visitor_activities: - type: object - $schema: http://json-schema.org/draft-07/schema# - additionalProperties: true - properties: - type: - type: - - "null" - - integer - campaign: - type: - - "null" - - object - campaign_id: - type: - - "null" - - integer - created_at: - type: - - "null" - - string - format: date-time - details: - type: - - "null" - - string - email_id: - type: - - "null" - - integer - email_template_id: - type: - - "null" - - integer - file_id: - type: - - "null" - - integer - form_handler_id: - type: - - "null" - - integer - form_id: - type: - - "null" - - integer - id: - type: - - integer - landing_page_id: - type: - - "null" - - integer - list_email_id: - type: - - "null" - - integer - multivariate_test_variation_id: - type: - - "null" - - integer - paid_search_id_id: - type: - - "null" - - integer - prospect_id: - type: - - "null" - - integer - site_search_query_id: - type: - - "null" - - integer - type_name: - type: - - "null" - - string - updated_at: - type: - - "null" - - string - format: date-time - visitor_id: - type: - - "null" - - integer - visitor_page_view_id: - type: - - "null" - - integer - visitors: - type: object - $schema: http://json-schema.org/draft-07/schema# - additionalProperties: true - properties: - campaign_parameter: - type: - - "null" - - string - content_parameter: - type: - - "null" - - string - created_at: - type: - - "null" - - string - format: date-time - hostname: - type: - - "null" - - string - id: - type: - - integer - ip_address: - type: - - "null" - - string - medium_parameter: - type: - - "null" - - string - page_view_count: - type: - - "null" - - integer - source_parameter: - type: - - "null" - - string - term_parameter: - type: - - "null" - - string - updated_at: - type: - - "null" - - string - format: date-time - visits: - type: object - $schema: http://json-schema.org/draft-07/schema# - additionalProperties: true - properties: - type: - type: - - "null" - - integer - campaign: - type: - - "null" - - object - campaign_parameter: - type: - - "null" - - string - content_parameter: - type: - - "null" - - string - created_at: - type: - - "null" - - string - format: date-time - details: - type: - - "null" - - string - duration_in_seconds: - type: - - "null" - - integer - email: - type: - - "null" - - object - email_id: - type: - - "null" - - integer - email_template_id: - type: - - "null" - - integer - first_visitor_page_view_at: - type: - - "null" - - string - format: date-time - id: - type: - - integer - last_visitor_page_view_at: - type: - - "null" - - string - format: date-time - list_email_id: - type: - - "null" - - integer - medium_parameter: - type: - - "null" - - string - prospect_id: - type: - - "null" - - integer - source_parameter: - type: - - "null" - - string - term_parameter: - type: - - "null" - - string - type_name: - type: - - "null" - - string - updated_at: - type: - - "null" - - string - format: date-time - visit_id: - type: - - "null" - - integer - visitor_id: - type: - - "null" - - integer - visitor_page_view_count: - type: - - "null" - - integer - visitor_page_views: - type: - - "null" - - object - opportunities: - type: object - $schema: http://json-schema.org/draft-07/schema# - additionalProperties: true - properties: - type: - type: - - "null" - - string - campaign_id: - type: - - "null" - - integer - closed_at: - type: - - "null" - - string - format: date-time - created_at: - type: - - "null" - - string - format: date-time - id: - type: - - integer - name: - type: - - "null" - - string - probability: - type: - - "null" - - integer - stage: - type: - - "null" - - string - status: - type: - - "null" - - string - updated_at: - type: - - "null" - - string - format: date-time - value: - type: - - "null" - - number +version: 6.10.0 + +type: DeclarativeSource + +check: + type: CheckStream + stream_names: + - campaigns + +definitions: + streams: + campaigns: + type: DeclarativeStream + name: campaigns + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/campaigns + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,name,isDeleted,folderId,cost,parentCampaignId,createdById,updatedById,createdAt,updatedAt,salesforceId + deleted: "{{ 'all' if not next_page_token.next_page_token }}" + orderBy: "{{ 'updatedAt ASC' if not next_page_token.next_page_token }}" + updatedAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + updatedAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updatedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%S-12:00" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "{{ config.split_up_interval }}" + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/campaigns" + email_clicks: + type: DeclarativeStream + name: email_clicks + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: emailClick/version/4/do/query + http_method: GET + request_parameters: + format: json + output: bulk + sort_by: id + sort_order: ascending + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - result + - emailClick + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: id_greater_than + page_size_option: + type: RequestOption + field_name: limit + inject_into: request_parameter + pagination_strategy: + type: CursorPagination + page_size: 200 + cursor_value: "{{ last_record.id }}" + stop_condition: >- + {{ not response.result.emailClick or + response.result.emailClick|length < 200 }} + incremental_sync: + type: DatetimeBasedCursor + cursor_field: created_at + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%d %H:%M:%S" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: created_after + inject_into: request_parameter + end_time_option: + type: RequestOption + field_name: created_before + inject_into: request_parameter + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "{{ config.split_up_interval }}" + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/email_clicks" + list_membership: + type: DeclarativeStream + name: list_membership + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/list-memberships + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,listId,prospectId,optedOut,createdAt,updatedAt,createdById,updatedById + deleted: "{{ 'all' if not next_page_token.next_page_token }}" + orderBy: "{{ 'updatedAt ASC' if not next_page_token.next_page_token }}" + updatedAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + updatedAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updatedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%S-12:00" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "{{ config.split_up_interval }}" + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/list_membership" + lists: + type: DeclarativeStream + name: lists + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/lists + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,name,title,description,isPublic,folderId,campaignId,isDeleted,isDynamic,createdAt,updatedAt,createdById,updatedById + deleted: "{{ 'all' if not next_page_token.next_page_token }}" + orderBy: "{{ 'updatedAt ASC' if not next_page_token.next_page_token }}" + updatedAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + updatedAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updatedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%S-12:00" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "{{ config.split_up_interval }}" + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/lists" + prospect_accounts: + type: DeclarativeStream + name: prospect_accounts + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/prospect-accounts + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,name,salesforceId,isDeleted,annualRevenue,billingAddressOne,billingAddressTwo,billingCity,billingCountry,billingState,billingZip,description,employees,fax,industry,number,ownership,phone,rating,shippingAddressOne,shippingAddressTwo,shippingCity,shippingCountry,shippingState,shippingZip,sic,site,tickerSymbol,type,website,createdAt,updatedAt,createdById,updatedById,assignedToId + deleted: "{{ 'all' if not next_page_token.next_page_token }}" + orderBy: "{{ 'id ASC' if not next_page_token.next_page_token }}" + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + partition_router: + type: ListPartitionRouter + values: + - "1" + cursor_field: >- + magic config: this is just a trick to make the next_page_token + object populate, which for some reason doesn't happen when only + Pagination is used (without Incremental or Param + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/prospect_accounts" + prospects: + type: DeclarativeStream + name: prospects + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/prospects + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,email,optedOut,isDeleted,isDoNotCall,isDoNotEmail,isEmailHardBounced,isReviewed,isStarred,doNotSell,prospectAccountId,campaignId,profileId,lifecycleStageId,userId,lastActivityAt,recentInteraction,firstName,lastName,salutation,jobTitle,emailBouncedAt,emailBouncedReason,source,sourceParameter,campaignParameter,mediumParameter,contentParameter,termParameter,firstActivityAt,firstAssignedAt,firstReferrerQuery,firstReferrerType,firstReferrerUrl,salesforceId,salesforceContactId,salesforceLeadId,salesforceAccountId,salesforceCampaignId,salesforceOwnerId,salesforceUrl,salesforceLastSync,assignedToId,convertedAt,convertedFromObjectName,convertedFromObjectType,grade,phone,fax,addressOne,addressTwo,city,state,zip,country,company,annualRevenue,website,industry,department,yearsInBusiness,employees,score,territory,comments,notes,createdAt,updatedAt,createdById,updatedById + deleted: "{{ 'all' if not next_page_token.next_page_token }}" + orderBy: "{{ 'updatedAt ASC' if not next_page_token.next_page_token }}" + updatedAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + updatedAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updatedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%S-12:00" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "{{ config.split_up_interval }}" + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/prospects" + users: + type: DeclarativeStream + name: users + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/users + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,email,username,isDeleted,firstName,jobTitle,role,roleName,salesforceId,tagReplacementLanguage,createdAt,updatedAt,createdById,updatedById + deleted: "{{ 'all' if not next_page_token.next_page_token }}" + orderBy: "{{ 'updatedAt ASC' if not next_page_token.next_page_token }}" + updatedAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + updatedAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updatedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%S-12:00" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "{{ config.split_up_interval }}" + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/users" + visitor_activities: + type: DeclarativeStream + name: visitor_activities + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/visitor-activities + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,visitId,prospectId,visitorId,emailId,type,typeName,details,campaignId,customRedirectId,emailTemplateId,fileId,formHandlerId,formId,landingPageId,listEmailId,multivariateTestVariationId,opportunityId,paidSearchAdId,siteSearchQueryId,visitorPageViewId,createdAt,updatedAt + orderBy: "{{ 'updatedAt ASC' if not next_page_token.next_page_token }}" + updatedAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + updatedAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updatedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%S-12:00" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "{{ config.split_up_interval }}" + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/visitor_activities" + visitors: + type: DeclarativeStream + name: visitors + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/visitors + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,prospectId,pageViewCount,isIdentified,doNotSell,hostname,ipAddress,campaignId,sourceParameter,campaignParameter,mediumParameter,contentParameter,termParameter,createdAt,updatedAt + deleted: "{{ 'all' if not next_page_token.next_page_token }}" + orderBy: "{{ 'updatedAt ASC' if not next_page_token.next_page_token }}" + updatedAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + updatedAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updatedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%S-12:00" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "{{ config.split_up_interval }}" + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/visitors" + visits: + type: DeclarativeStream + name: visits + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/visits + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,visitorId,prospectId,visitorPageViewCount,firstVisitorPageViewAt,lastVisitorPageViewAt,durationInSeconds,sourceParameter,campaignParameter,mediumParameter,contentParameter,termParameter,createdAt,updatedAt + orderBy: "{{ 'updatedAt ASC' if not next_page_token.next_page_token }}" + updatedAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + updatedAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updatedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%S-12:00" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "{{ config.split_up_interval }}" + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/visits" + folders: + type: DeclarativeStream + name: folders + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/folders + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,name,parentFolderId,path,usePermissions,createdAt,updatedAt,createdById,updatedById + orderBy: "{{ 'id ASC' if not next_page_token.next_page_token }}" + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + partition_router: + type: ListPartitionRouter + values: + - "1" + cursor_field: >- + magic config: this is just a trick to make the next_page_token + object populate, which for some reason doesn't happen when only + Pagination is used (without Incremental or Param + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/folders" + custom_redirects: + type: DeclarativeStream + name: custom_redirects + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/custom-redirects + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,name,url,destinationUrl,vanityUrl,campaignId,salesforceId,isDeleted,folderId,trackerDomainId,trackerDomain.domain,vanityUrlPath,trackedUrl,bitlyIsPersonalized,bitlyShortUrl,gaSource,gaMedium,gaTerm,gaContent,gaCampaign,createdAt,updatedAt,createdById,updatedById + deleted: "{{ 'all' if not next_page_token.next_page_token }}" + orderBy: "{{ 'id ASC' if not next_page_token.next_page_token }}" + updatedAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + updatedAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/custom_redirects" + emails: + type: DeclarativeStream + name: emails + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/emails + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,name,type,sentAt,subject,clientType,isOperational,campaignId,prospectId,folderId,listId,listEmailId,emailTemplateId,trackerDomainId,senderOptions.type,senderOptions.address,senderOptions.name,senderOptions.userId,senderOptions.prospectCustomFieldId,senderOptions.accountCustomFieldId,replyToOptions.type,replyToOptions.address,replyToOptions.userId,replyToOptions.prospectCustomFieldId,replyToOptions.accountCustomFieldId,createdById + orderBy: "{{ 'sentAt ASC' if not next_page_token.next_page_token }}" + sentAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + sentAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: sentAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%S-12:00" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: P1D + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/emails" + engagement_studio_programs: + type: DeclarativeStream + name: engagement_studio_programs + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/engagement-studio-programs + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,status,isDeleted,salesforceId,description,businessHours,prospectsMultipleEntry,schedule,scheduleCreatedById,recipientListIds,suppressionListIds,createdAt,updatedAt,createdById,updatedById + deleted: "{{ 'all' if not next_page_token.next_page_token }}" + orderBy: "{{ 'updatedAt ASC' if not next_page_token.next_page_token }}" + updatedAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + updatedAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updatedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%S-12:00" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "{{ config.split_up_interval }}" + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/engagement_studio_programs" + files: + type: DeclarativeStream + name: files + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/files + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,name,folderId,campaignId,salesforceId,trackerDomainId,vanityUrl,vanityUrlPath,url,size,isTracked,bitlyIsPersonalized,bitlyShortUrl,createdAt,updatedAt,createdById,updatedById + orderBy: "{{ 'id ASC' if not next_page_token.next_page_token }}" + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/files" + folder_contents: + type: DeclarativeStream + name: folder_contents + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/folder-contents + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,folderId,folderRef,objectType,objectId,objectName,objectRef,createdAt,updatedAt,createdById,updatedById + orderBy: "{{ 'updatedAt ASC' if not next_page_token.next_page_token }}" + updatedAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + updatedAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updatedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%S-12:00" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "{{ config.split_up_interval }}" + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/folder_contents" + forms: + type: DeclarativeStream + name: forms + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/forms + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,name,campaignId,layoutTemplateId,folderId,trackerDomainId,salesforceId,isDeleted,isUseRedirectLocation,isAlwaysDisplay,isCookieless,isCaptchaEnabled,showNotProspect,embedCode,submitButtonText,beforeFormContent,afterFormContent,thankYouContent,thankYouCode,redirectLocation,fontSize,fontFamily,fontColor,labelAlignment,radioAlignment,checkboxAlignment,requiredCharacter,createdById,updatedById,createdAt,updatedAt + deleted: "{{ 'all' if not next_page_token.next_page_token }}" + orderBy: "{{ 'id ASC' if not next_page_token.next_page_token }}" + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/forms" + form_fields: + type: DeclarativeStream + name: form_fields + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/form-fields + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,formId,prospectApiFieldId,type,dataFormat,sortOrder,hasDependents,hasProgressives,hasValues,label,errorMessage,cssClasses,isRequired,isAlwaysDisplay,isMaintainInitialValue,isDoNotPrefill,createdById,updatedById,createdAt,updatedAt + orderBy: "{{ 'updatedAt ASC' if not next_page_token.next_page_token }}" + updatedAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + updatedAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updatedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%S-12:00" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "{{ config.split_up_interval }}" + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/form_fields" + form_handlers: + type: DeclarativeStream + name: form_handlers + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/form-handlers + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,name,folderId,campaignId,trackerDomainId,isDataForwarded,successLocation,errorLocation,isAlwaysEmail,isCookieless,salesforceId,embedCode,createdAt,createdById,isDeleted,updatedById + deleted: "{{ 'all' if not next_page_token.next_page_token }}" + orderBy: "{{ 'id ASC' if not next_page_token.next_page_token }}" + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/form_handlers" + form_handler_fields: + type: DeclarativeStream + name: form_handler_fields + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/form-handler-fields + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,name,formHandlerId,dataFormat,prospectApiFieldId,isMaintainInitialValue,errorMessage,isRequired,createdAt,createdById + orderBy: "{{ 'id ASC' if not next_page_token.next_page_token }}" + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/form_handler_fields" + landing_pages: + type: DeclarativeStream + name: landing_pages + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/landing-pages + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,name,campaignId,salesforceId,isDeleted,layoutType,layoutTableBorder,isUseRedirectLocation,bitlyIsPersonalized,bitlyShortUrl,url,vanityUrl,folderId,formId,layoutTemplateId,title,description,isDoNotIndex,vanityUrlPath,redirectLocation,trackerDomainId,archiveDate,createdAt,updatedAt,createdById,updatedById + deleted: "{{ 'all' if not next_page_token.next_page_token }}" + orderBy: "{{ 'updatedAt ASC' if not next_page_token.next_page_token }}" + updatedAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + updatedAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updatedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%S-12:00" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "{{ config.split_up_interval }}" + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/landing_pages" + layout_templates: + type: DeclarativeStream + name: layout_templates + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/layout-templates + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,name,layoutContent,siteSearchContent,formContent,folderId,isDeleted,isIncludeDefaultCss,createdAt,updatedAt,createdById,updatedById + deleted: "{{ 'all' if not next_page_token.next_page_token }}" + orderBy: "{{ 'id ASC' if not next_page_token.next_page_token }}" + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/layout_templates" + lifecycle_stages: + type: DeclarativeStream + name: lifecycle_stages + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/lifecycle-stages + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: id,name,isDeleted,isLocked,position,matchType,createdAt,updatedAt + deleted: "{{ 'all' if not next_page_token.next_page_token }}" + orderBy: "{{ 'updatedAt ASC' if not next_page_token.next_page_token }}" + updatedAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + updatedAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updatedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%S-12:00" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "{{ config.split_up_interval }}" + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/lifecycle_stages" + lifecycle_histories: + type: DeclarativeStream + name: lifecycle_histories + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/lifecycle-histories + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: id,prospectId,previousStageId,nextStageId,secondsElapsed,createdAt + orderBy: "{{ 'createdAt ASC' if not next_page_token.next_page_token }}" + createdAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + createdAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: createdAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%S-12:00" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "{{ config.split_up_interval }}" + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/lifecycle_histories" + list_emails: + type: DeclarativeStream + name: list_emails + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/list-emails + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,name,subject,isPaused,isSent,isDeleted,isOperational,sentAt,campaignId,clientType,senderOptions.type,senderOptions.address,senderOptions.name,senderOptions.userId,senderOptions.prospectCustomFieldId,senderOptions.accountCustomFieldId,replyToOptions.type,replyToOptions.address,replyToOptions.userId,replyToOptions.prospectCustomFieldId,replyToOptions.accountCustomFieldId,emailTemplateId,trackerDomainId,folderId,createdAt,updatedAt,createdById,updatedById + deleted: "{{ 'all' if not next_page_token.next_page_token }}" + orderBy: "{{ 'updatedAt ASC' if not next_page_token.next_page_token }}" + updatedAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + updatedAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updatedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%S-12:00" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "{{ config.split_up_interval }}" + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/list_emails" + opportunities: + type: DeclarativeStream + name: opportunities + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/opportunities + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,closedAt,name,type,stage,status,probability,value,campaignId,salesforceId,createdAt,updatedAt,createdById,updatedById + orderBy: "{{ 'updatedAt ASC' if not next_page_token.next_page_token }}" + updatedAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + updatedAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updatedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%S-12:00" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "{{ config.split_up_interval }}" + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/opportunities" + tags: + type: DeclarativeStream + name: tags + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/tags + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: id,name,objectCount,createdById,updatedById,createdAt,updatedAt + orderBy: "{{ 'updatedAt ASC' if not next_page_token.next_page_token }}" + updatedAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + updatedAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updatedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%S-12:00" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "{{ config.split_up_interval }}" + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/tags" + tracker_domains: + type: DeclarativeStream + name: tracker_domains + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/tracker-domains + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,domain,isPrimary,isDeleted,defaultCampaignId,httpsStatus,sslStatus,sslStatusDetails,sslRequestedById,validationStatus,validatedAt,vanityUrlStatus,trackingCode,createdAt,updatedAt,createdById,updatedById + deleted: "{{ 'all' if not next_page_token.next_page_token }}" + orderBy: "{{ 'id ASC' if not next_page_token.next_page_token }}" + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/tracker_domains" + visitor_page_views: + type: DeclarativeStream + name: visitor_page_views + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/visitor-page-views + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,url,title,visitorId,campaignId,visitId,durationInSeconds,salesforceId,createdAt + orderBy: "{{ 'createdAt ASC' if not next_page_token.next_page_token }}" + createdAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + createdAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: createdAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%S-12:00" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "{{ config.split_up_interval }}" + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/visitor_page_views" + account: + type: DeclarativeStream + name: account + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/account + http_method: GET + request_parameters: + fields: >- + id,company,level,website,pluginCampaignId,addressOne,addressTwo,city,state,zip,territory,country,phone,fax,adminId,maximumDailyApiCalls,apiCallsUsed,createdAt,updatedAt,createdById,updatedById + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/account" + custom_fields: + type: DeclarativeStream + name: custom_fields + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/custom-fields + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,name,fieldId,type,salesforceId,isRequired,isRecordMultipleResponses,isUseValues,isAnalyticsSynced,createdAt,updatedAt,createdById,updatedById + orderBy: "{{ 'updatedAt ASC' if not next_page_token.next_page_token }}" + updatedAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + updatedAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updatedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%S-12:00" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "{{ config.split_up_interval }}" + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/custom_fields" + dynamic_contents: + type: DeclarativeStream + name: dynamic_contents + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/dynamic-contents + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: >- + id,name,basedOnProspectApiFieldId,embedCode,embedUrl,basedOn,tagReplacementLanguage,folderId,trackerDomainId,createdAt,updatedAt,isDeleted,createdById,updatedById + deleted: "{{ 'all' if not next_page_token.next_page_token }}" + orderBy: "{{ 'updatedAt ASC' if not next_page_token.next_page_token }}" + updatedAtAfterOrEqualTo: >- + {{ stream_interval.start_time if stream_interval.start_time and + not next_page_token.next_page_token }} + updatedAtBeforeOrEqualTo: >- + {{ stream_interval.end_time if stream_interval.end_time and not + next_page_token.next_page_token }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updatedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%S-12:00" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.start_date }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "{{ config.split_up_interval }}" + cursor_granularity: PT1S + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/dynamic_contents" + dynamic_content_variations: + type: DeclarativeStream + name: dynamic_content_variations + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: v5/objects/dynamic-content-variations + http_method: GET + request_parameters: + limit: "{{ config.page_size if not next_page_token.next_page_token }}" + fields: id,dynamicContentId,comparison,operator,value1,value2,content + orderBy: "{{ 'id ASC' if not next_page_token.next_page_token }}" + dynamicContentId: >- + {{ stream_partition.parent_id.dynamic_content_id if + stream_interval.start_time and not next_page_token.next_page_token + }} + request_headers: + Pardot-Business-Unit-Id: "{{ config.pardot_business_unit_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - values + schema_normalization: Default + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.get(\"nextPageUrl\", {}) }}" + stop_condition: "{{ not response.get(\"nextPageUrl\", {}) }}" + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: dynamic_content_id + stream: + $ref: "#/definitions/streams/dynamic_contents" + incremental_dependency: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/dynamic_content_variations" + base_requester: + type: HttpRequester + url_base: https://pi.pardot.com/api/ + authenticator: + type: SessionTokenAuthenticator + login_requester: + type: HttpRequester + url_base: >- + https://{{ 'test' if config['is_sandbox'] else + 'login'}}.salesforce.com/services/oauth2 + path: token + authenticator: + type: NoAuth + http_method: POST + request_parameters: {} + request_headers: {} + request_body_data: + client_id: "'{{ config.client_id }}'" + grant_type: refresh_token + client_secret: "'{{ config.client_secret }}'" + refresh_token: "'{{ config.refresh_token }}'" + session_token_path: + - access_token + expiration_duration: PT24H + request_authentication: + type: Bearer + +streams: + - $ref: "#/definitions/streams/campaigns" + - $ref: "#/definitions/streams/email_clicks" + - $ref: "#/definitions/streams/list_membership" + - $ref: "#/definitions/streams/lists" + - $ref: "#/definitions/streams/prospect_accounts" + - $ref: "#/definitions/streams/prospects" + - $ref: "#/definitions/streams/users" + - $ref: "#/definitions/streams/visitor_activities" + - $ref: "#/definitions/streams/visitors" + - $ref: "#/definitions/streams/visits" + - $ref: "#/definitions/streams/folders" + - $ref: "#/definitions/streams/custom_redirects" + - $ref: "#/definitions/streams/emails" + - $ref: "#/definitions/streams/engagement_studio_programs" + - $ref: "#/definitions/streams/files" + - $ref: "#/definitions/streams/folder_contents" + - $ref: "#/definitions/streams/forms" + - $ref: "#/definitions/streams/form_fields" + - $ref: "#/definitions/streams/form_handlers" + - $ref: "#/definitions/streams/form_handler_fields" + - $ref: "#/definitions/streams/landing_pages" + - $ref: "#/definitions/streams/layout_templates" + - $ref: "#/definitions/streams/lifecycle_stages" + - $ref: "#/definitions/streams/lifecycle_histories" + - $ref: "#/definitions/streams/list_emails" + - $ref: "#/definitions/streams/opportunities" + - $ref: "#/definitions/streams/tags" + - $ref: "#/definitions/streams/tracker_domains" + - $ref: "#/definitions/streams/visitor_page_views" + - $ref: "#/definitions/streams/account" + - $ref: "#/definitions/streams/custom_fields" + - $ref: "#/definitions/streams/dynamic_contents" + - $ref: "#/definitions/streams/dynamic_content_variations" + +spec: + type: Spec + connection_specification: + type: object + $schema: http://json-schema.org/draft-07/schema# + required: + - pardot_business_unit_id + - client_id + - client_secret + - refresh_token + properties: + pardot_business_unit_id: + type: string + description: >- + Pardot Business ID, can be found at Setup > Pardot > Pardot Account + Setup + order: 0 + title: Pardot Business Unit ID + client_id: + type: string + description: The Consumer Key that can be found when viewing your app in Salesforce + order: 1 + title: Client ID + airbyte_secret: true + client_secret: + type: string + description: >- + The Consumer Secret that can be found when viewing your app in + Salesforce + order: 2 + title: Client Secret + airbyte_secret: true + refresh_token: + type: string + description: >- + Salesforce Refresh Token used for Airbyte to access your Salesforce + account. If you don't know what this is, follow this <a + href="https://medium.com/@bpmmendis94/obtain-access-refresh-tokens-from-salesforce-rest-api-a324fe4ccd9b">guide</a> + to retrieve it. + order: 3 + title: Refresh Token + airbyte_secret: true + start_date: + type: string + description: >- + UTC date and time in the format 2000-01-01T00:00:00Z. Any data before + this date will not be replicated. Defaults to the year Pardot was + released. + order: 4 + title: Start Date + format: date-time + default: "2007-01-01T00:00:00Z" + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ + examples: + - "2021-07-25T00:00:00Z" + page_size: + type: string + description: The maximum number of records to return per request + order: 5 + title: Page Size Limit + default: "1000" + is_sandbox: + type: boolean + description: >- + Whether or not the the app is in a Salesforce sandbox. If you do not + know what this, assume it is false. + order: 6 + title: Is Sandbox App? + default: false + split_up_interval: + type: string + description: >- + If you're hitting the max pagination limit of the v5 API (100K), try + choosing a more granular value (e.g. P7D). Similarly for small + accounts unlikely to hit this limit, a less granular value (e.g. P1Y) + may speed up the sync. + enum: + - P1Y + - P6M + - P3M + - P1M + - P14D + - P7D + - P3D + - P1D + order: 7 + title: Default Split Up Interval + default: P3M + additionalProperties: true + +metadata: + autoImportSchema: + campaigns: false + email_clicks: false + list_membership: false + lists: false + prospect_accounts: false + prospects: false + users: false + visitor_activities: false + visitors: false + visits: false + folders: false + custom_redirects: false + emails: false + engagement_studio_programs: false + files: false + folder_contents: false + forms: false + form_fields: false + form_handlers: false + form_handler_fields: false + landing_pages: false + layout_templates: false + lifecycle_stages: false + lifecycle_histories: false + list_emails: false + opportunities: false + tags: false + tracker_domains: false + visitor_page_views: false + account: false + custom_fields: false + dynamic_contents: false + dynamic_content_variations: false + testedStreams: + campaigns: + hasRecords: true + streamHash: fc1d627201b08eec5c250804ed7dcd7f09f9675c + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + email_clicks: + hasRecords: true + streamHash: 411f4328e02a6d5ce0a3ec3a9a3cb54804e374e0 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + list_membership: + hasRecords: true + streamHash: e3d9c46036a8c5281ad1cec8c9a1b1246ae6df97 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + lists: + hasRecords: true + streamHash: d6ee1f6521a0fc63d809c213ea53613d50d4c178 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + prospect_accounts: + hasRecords: true + streamHash: 37a2ac73f3e6564eba2aad749645c81ee4b7e8ba + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + prospects: + hasRecords: true + streamHash: eaf5ca07271f45b4128dcb4e6da41df2dd853f5d + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + users: + hasRecords: true + streamHash: bfaff23353daea243780a306ede24a6abc60ded3 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + visitor_activities: + hasRecords: true + streamHash: 8544f58fbc35ed2e59f62caf60e6fdf7224d7cf8 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + visitors: + hasRecords: true + streamHash: 2fdf03a97b7ae2186d2a32a3dcae26abd2b6d06c + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + visits: + hasRecords: true + streamHash: 788619013dc2ed5e75320de887b02d3f0ffc0495 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + folders: + hasRecords: true + streamHash: 952ec71d864df0763936fcf9a48ca5ee2a3f9037 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + custom_redirects: + hasRecords: false + streamHash: 73ca21e4c8acb6ca7a4efad4517b31af4dfcb6f6 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + emails: + hasRecords: true + streamHash: 25e6dc61f25fb81825b764af684379d04e748854 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + engagement_studio_programs: + hasRecords: true + streamHash: 2108ab2546a6df889a2420e2d8cb3aac4b790b70 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + files: + hasRecords: false + streamHash: 9dd470ef4814f6fab7a1acf61671ef171b839da2 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + folder_contents: + hasRecords: true + streamHash: 80dc85cda30756bb57fb750432ad8e580d32a245 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + forms: + hasRecords: true + streamHash: a8143b403bf769d4d24dc771da7888a45cc8cf0c + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + form_fields: + hasRecords: true + streamHash: 6f148b11889ab260145dd5e865a142a85ee35d9f + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + form_handlers: + hasRecords: true + streamHash: 8621f40695e6f14df6b5ac1c5c90f6334fc5866a + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + form_handler_fields: + hasRecords: true + streamHash: 1827a46a501b04b0c21e6411d464c77a08dfb848 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + landing_pages: + hasRecords: true + streamHash: fc06118614b8de9f2c2c53e6c5bbe88745e28c55 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + layout_templates: + hasRecords: true + streamHash: 9088e72d125495ef1d72d881a8c6b72e4950e06d + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + lifecycle_stages: + hasRecords: true + streamHash: c0eab225bad7533145cfeac38e57bb70745f9a79 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + lifecycle_histories: + hasRecords: true + streamHash: 21a5786703917ee27352156ea80fa93cce846954 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + list_emails: + streamHash: 1823761c89debf594562433f63c8661b3be55524 + hasResponse: true + responsesAreSuccessful: true + hasRecords: true + primaryKeysArePresent: true + primaryKeysAreUnique: true + opportunities: + streamHash: 0def746ae5fd7dde2e85858a1be8b72501dca4d5 + hasResponse: true + responsesAreSuccessful: true + hasRecords: true + primaryKeysArePresent: true + primaryKeysAreUnique: true + tags: + streamHash: 4ce0030af2ff46782dfdb5a1bea97e8b92e2a493 + hasResponse: true + responsesAreSuccessful: true + hasRecords: true + primaryKeysArePresent: true + primaryKeysAreUnique: true + tracker_domains: + streamHash: e441705cc25b37d24b9163afac9e22a734f2185f + hasResponse: true + responsesAreSuccessful: true + hasRecords: true + primaryKeysArePresent: true + primaryKeysAreUnique: true + visitor_page_views: + streamHash: 561cc8d52b41daa6f4697d2e2a3efdee3cc3a14d + hasResponse: true + responsesAreSuccessful: true + hasRecords: true + primaryKeysArePresent: true + primaryKeysAreUnique: true + account: + streamHash: 0893578c116978dad6459ef40fe2a52ea26a68e9 + hasResponse: true + responsesAreSuccessful: false + hasRecords: false + primaryKeysArePresent: true + primaryKeysAreUnique: true + custom_fields: + streamHash: 020b57ff80114642d70f122a63b22f05d63d8386 + hasResponse: true + responsesAreSuccessful: false + hasRecords: false + primaryKeysArePresent: true + primaryKeysAreUnique: true + dynamic_contents: + streamHash: 795500f19349e89d320fb4814aedf19f5bff9dff + hasResponse: true + responsesAreSuccessful: true + hasRecords: false + primaryKeysArePresent: true + primaryKeysAreUnique: true + dynamic_content_variations: + streamHash: null + assist: {} + +schemas: + campaigns: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + cost: + type: + - "null" + - integer + id: + type: + - integer + name: + type: + - "null" + - string + email_clicks: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + created_at: + type: + - "null" + - string + format: date-time + drip_program_action_id: + type: + - "null" + - integer + email_template_id: + type: + - "null" + - integer + id: + type: + - integer + list_email_id: + type: + - "null" + - integer + prospect_id: + type: + - "null" + - integer + tracker_redirect_id: + type: + - "null" + - integer + url: + type: + - "null" + - string + list_membership: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + createdAt: + type: + - "null" + - string + format: date-time + createdById: + type: + - "null" + - integer + id: + type: + - integer + listId: + type: + - integer + optedOut: + type: + - "null" + - boolean + prospectId: + type: + - integer + updatedAt: + type: + - "null" + - string + format: date-time + updatedById: + type: + - "null" + - integer + required: + - id + - updatedAt + lists: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + description: + type: + - "null" + - string + createdAt: + type: + - "null" + - string + format: date-time + createdById: + type: + - "null" + - integer + folderId: + type: + - "null" + - integer + id: + type: + - integer + isDeleted: + type: + - "null" + - boolean + isDynamic: + type: + - "null" + - boolean + isPublic: + type: + - "null" + - boolean + name: + type: + - "null" + - string + title: + type: + - "null" + - string + updatedAt: + type: + - "null" + - string + format: date-time + updatedById: + type: + - "null" + - integer + prospect_accounts: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - "null" + - string + description: + type: + - "null" + - string + annualRevenue: + type: + - "null" + - string + assignedToId: + type: + - "null" + - integer + billingAddressOne: + type: + - "null" + - string + billingAddressTwo: + type: + - "null" + - string + billingCity: + type: + - "null" + - string + billingCountry: + type: + - "null" + - string + billingState: + type: + - "null" + - string + billingZip: + type: + - "null" + - string + createdAt: + type: + - "null" + - string + format: date-time + createdById: + type: + - "null" + - integer + employees: + type: + - "null" + - string + fax: + type: + - "null" + - string + id: + type: + - integer + industry: + type: + - "null" + - string + isDeleted: + type: + - "null" + - boolean + name: + type: + - "null" + - string + number: + type: + - "null" + - string + ownership: + type: + - "null" + - string + phone: + type: + - "null" + - string + rating: + type: + - "null" + - string + salesforceId: + type: + - "null" + - string + shippingAddressOne: + type: + - "null" + - string + shippingAddressTwo: + type: + - "null" + - string + shippingCity: + type: + - "null" + - string + shippingCountry: + type: + - "null" + - string + shippingState: + type: + - "null" + - string + shippingZip: + type: + - "null" + - string + sic: + type: + - "null" + - string + site: + type: + - "null" + - string + tickerSymbol: + type: + - "null" + - string + updatedAt: + type: + - "null" + - string + format: date-time + updatedById: + type: + - "null" + - integer + website: + type: + - "null" + - string + prospects: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + addressOne: + type: + - "null" + - string + addressTwo: + type: + - "null" + - string + annualRevenue: + type: + - "null" + - string + assignedToId: + type: + - "null" + - string + campaignId: + type: + - "null" + - integer + campaignParameter: + type: + - "null" + - string + city: + type: + - "null" + - string + comments: + type: + - "null" + - string + company: + type: + - "null" + - string + contentParameter: + type: + - "null" + - string + convertedAt: + type: + - "null" + - string + format: date-time + convertedFromObjectName: + type: + - "null" + - string + convertedFromObjectType: + type: + - "null" + - string + country: + type: + - "null" + - string + createdAt: + type: + - "null" + - string + format: date-time + createdById: + type: + - "null" + - integer + department: + type: + - "null" + - string + doNotSell: + type: + - "null" + - boolean + email: + type: + - "null" + - string + emailBouncedAt: + type: + - "null" + - string + format: date-time + emailBouncedReason: + type: + - "null" + - string + employees: + type: + - "null" + - string + fax: + type: + - "null" + - string + firstActivityAt: + type: + - "null" + - string + format: date-time + firstAssignedAt: + type: + - "null" + - string + format: date-time + firstName: + type: + - "null" + - string + firstReferrerQuery: + type: + - "null" + - string + firstReferrerType: + type: + - "null" + - string + firstReferrerUrl: + type: + - "null" + - string + grade: + type: + - "null" + - string + id: + type: + - integer + industry: + type: + - "null" + - string + isDeleted: + type: + - "null" + - boolean + isDoNotCall: + type: + - "null" + - boolean + isDoNotEmail: + type: + - "null" + - boolean + isEmailHardBounced: + type: + - "null" + - boolean + isReviewed: + type: + - "null" + - boolean + isStarred: + type: + - "null" + - boolean + jobTitle: + type: + - "null" + - string + lastActivityAt: + type: + - "null" + - string + format: date-time + lastName: + type: + - "null" + - string + lifecycleStageId: + type: + - "null" + - integer + mediumParameter: + type: + - "null" + - string + notes: + type: + - "null" + - string + optedOut: + type: + - "null" + - boolean + phone: + type: + - "null" + - string + profileId: + type: + - "null" + - integer + prospectAccountId: + type: + - "null" + - integer + recentInteraction: + type: + - "null" + - string + salesforceAccountId: + type: + - "null" + - string + salesforceCampaignId: + type: + - "null" + - string + salesforceContactId: + type: + - "null" + - string + salesforceId: + type: + - "null" + - string + salesforceLastSync: + type: + - "null" + - string + format: date-time + salesforceLeadId: + type: + - "null" + - string + salesforceOwnerId: + type: + - "null" + - string + salesforceUrl: + type: + - "null" + - string + salutation: + type: + - "null" + - string + score: + type: + - "null" + - string + source: + type: + - "null" + - string + sourceParameter: + type: + - "null" + - string + state: + type: + - "null" + - string + termParameter: + type: + - "null" + - string + territory: + type: + - "null" + - string + updatedAt: + type: + - "null" + - string + format: date-time + updatedById: + type: + - "null" + - integer + userId: + type: + - "null" + - integer + website: + type: + - "null" + - string + yearsInBusiness: + type: + - "null" + - string + zip: + type: + - "null" + - string + users: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + createdAt: + type: + - "null" + - string + format: date-time + createdById: + type: + - "null" + - integer + email: + type: + - "null" + - string + firstName: + type: + - "null" + - string + id: + type: + - integer + isDeleted: + type: + - "null" + - boolean + jobTitle: + type: + - "null" + - string + role: + type: + - "null" + - string + roleName: + type: + - "null" + - string + salesforceId: + type: + - "null" + - string + tagReplacementLanguage: + type: + - "null" + - string + updatedAt: + type: + - "null" + - string + format: date-time + updatedById: + type: + - "null" + - integer + username: + type: + - "null" + - string + visitor_activities: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - "null" + - integer + campaignId: + type: + - "null" + - integer + createdAt: + type: + - "null" + - string + format: date-time + customRedirectId: + type: + - "null" + - integer + details: + type: + - "null" + - string + emailId: + type: + - "null" + - integer + emailTemplateId: + type: + - "null" + - integer + fileId: + type: + - "null" + - integer + formHandlerId: + type: + - "null" + - integer + formId: + type: + - "null" + - integer + id: + type: + - integer + landingPageId: + type: + - "null" + - integer + listEmailId: + type: + - "null" + - integer + multivariateTestVariationId: + type: + - "null" + - integer + opportunityId: + type: + - "null" + - integer + paidSearchAdId: + type: + - "null" + - integer + prospectId: + type: + - "null" + - integer + siteSearchQueryId: + type: + - "null" + - integer + typeName: + type: + - "null" + - string + updatedAt: + type: + - "null" + - string + format: date-time + visitId: + type: + - "null" + - integer + visitorId: + type: + - "null" + - integer + visitorPageViewId: + type: + - "null" + - integer + visitors: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + campaignId: + type: + - "null" + - integer + campaignParameter: + type: + - "null" + - string + contentParameter: + type: + - "null" + - string + createdAt: + type: + - "null" + - string + format: date-time + doNotSell: + type: + - "null" + - boolean + hostname: + type: + - "null" + - string + id: + type: + - integer + ipAddress: + type: + - "null" + - string + isIdentified: + type: + - "null" + - boolean + mediumParameter: + type: + - "null" + - string + pageViewCount: + type: + - "null" + - integer + prospectId: + type: + - "null" + - integer + sourceParameter: + type: + - "null" + - string + termParameter: + type: + - "null" + - string + updatedAt: + type: + - "null" + - string + format: date-time + visits: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + campaignParameter: + type: + - "null" + - string + contentParameter: + type: + - "null" + - string + createdAt: + type: + - "null" + - string + format: date-time + durationInSeconds: + type: + - "null" + - integer + firstVisitorPageViewAt: + type: + - "null" + - string + format: date-time + id: + type: + - integer + lastVisitorPageViewAt: + type: + - "null" + - string + format: date-time + mediumParameter: + type: + - "null" + - string + prospectId: + type: + - "null" + - integer + sourceParameter: + type: + - "null" + - string + termParameter: + type: + - "null" + - string + updatedAt: + type: + - "null" + - string + format: date-time + visitorId: + type: + - "null" + - integer + visitorPageViewCount: + type: + - "null" + - integer + folders: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + createdAt: + type: + - "null" + - string + createdById: + type: + - "null" + - integer + id: + type: integer + name: + type: + - "null" + - string + parentFolderId: + type: + - "null" + - integer + path: + type: + - "null" + - string + updatedAt: + type: + - "null" + - string + updatedById: + type: + - "null" + - integer + usePermissions: + type: + - "null" + - boolean + required: + - id + custom_redirects: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + bitlyIsPersonalized: + type: + - "null" + - boolean + bitlyShortUrl: + type: + - "null" + - string + campaignId: + type: + - "null" + - integer + createdAt: + type: + - "null" + - string + createdById: + type: + - "null" + - integer + destinationUrl: + type: + - "null" + - string + folderId: + type: + - "null" + - integer + gaCampaign: + type: + - "null" + - string + gaContent: + type: + - "null" + - string + gaMedium: + type: + - "null" + - string + gaSource: + type: + - "null" + - string + gaTerm: + type: + - "null" + - string + id: + type: integer + isDeleted: + type: + - "null" + - boolean + name: + type: + - "null" + - string + salesforceId: + type: + - "null" + - integer + trackedUrl: + type: + - "null" + - string + trackerDomain.domain: + type: + - "null" + - string + trackerDomainId: + type: + - "null" + - integer + updatedAt: + type: + - "null" + - string + updatedById: + type: + - "null" + - integer + url: + type: + - "null" + - string + vanityUrl: + type: + - "null" + - string + vanityUrlPath: + type: + - "null" + - string + required: + - id + emails: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + campaignId: + type: + - integer + - "null" + clientType: + type: + - string + - "null" + createdById: + type: + - integer + - "null" + emailTemplateId: + type: + - integer + - "null" + id: + type: integer + isOperational: + type: + - boolean + - "null" + listEmailId: + type: + - integer + - "null" + name: + type: + - string + - "null" + prospectId: + type: + - integer + - "null" + replyToOptions: + type: + - array + - "null" + items: + type: + - object + - "null" + properties: + type: + type: + - string + - "null" + accountCustomFieldId: + type: + - integer + - "null" + address: + type: + - string + - "null" + name: + type: + - string + - "null" + prospectCustomFieldId: + type: + - integer + - "null" + userId: + type: + - integer + - "null" + senderOptions: + type: + - array + - "null" + items: + type: + - object + - "null" + properties: + type: + type: + - string + - "null" + accountCustomFieldId: + type: + - integer + - "null" + address: + type: + - string + - "null" + name: + type: + - string + - "null" + prospectCustomFieldId: + type: + - integer + - "null" + userId: + type: + - integer + - "null" + sentAt: + type: + - "null" + - string + format: date-time + subject: + type: + - string + - "null" + required: + - id + - sentAt + engagement_studio_programs: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + description: + type: + - string + - "null" + businessHours: + type: + - object + - "null" + properties: + days: + type: + - array + - "null" + items: + type: + - string + - "null" + endTime: + type: + - string + - "null" + airbyte_type: time_without_timezone + format: time + startTime: + type: + - string + - "null" + airbyte_type: time_without_timezone + format: time + timezone: + type: + - string + - "null" + createdAt: + type: + - string + - "null" + format: date-time + createdById: + type: + - integer + - "null" + id: + type: integer + isDeleted: + type: + - boolean + - "null" + prospectsMultipleEntry: + type: + - object + - "null" + properties: + maximumEntries: + type: + - integer + - "null" + minimumDurationInDays: + type: + - integer + - "null" + recipientListIds: + type: + - array + - "null" + items: + type: + - integer + - "null" + salesforceId: + type: + - string + - "null" + schedule: + type: + - object + - "null" + properties: + createdAt: + type: + - string + - "null" + format: date-time + startOn: + type: + - string + - "null" + format: date-time + stopOn: + type: + - string + - "null" + format: date-time + scheduleCreatedById: + type: + - integer + - "null" + status: + type: + - string + - "null" + suppressionListIds: + type: + - array + - "null" + items: + type: + - integer + - "null" + updatedAt: + type: + - string + - "null" + format: date-time + updatedById: + type: + - integer + - "null" + required: + - id + - updatedAt + files: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + bitlyIsPersonalized: + type: + - boolean + - "null" + bitlyShortUrl: + type: + - string + - "null" + campaignId: + type: + - integer + - "null" + createdAt: + type: + - string + - "null" + format: date-time + createdById: + type: + - integer + - "null" + folderId: + type: + - integer + - "null" + id: + type: integer + isTracked: + type: + - boolean + - "null" + name: + type: + - string + - "null" + salesforceId: + type: + - string + - "null" + size: + type: + - integer + - "null" + trackerDomainId: + type: + - integer + - "null" + updatedAt: + type: + - string + - "null" + format: date-time + updatedById: + type: + - integer + - "null" + url: + type: + - string + - "null" + vanityUrl: + type: + - string + - "null" + vanityUrlPath: + type: + - string + - "null" + required: + - id + folder_contents: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + createdAt: + type: + - string + - "null" + format: date-time + createdById: + type: + - integer + - "null" + folderId: + type: + - integer + - "null" + folderRef: + type: + - string + - "null" + id: + type: integer + objectId: + type: + - integer + - "null" + objectName: + type: + - string + - "null" + objectRef: + type: + - string + - "null" + objectType: + type: + - string + - "null" + updatedAt: + type: + - string + - "null" + format: date-time + updatedById: + type: + - integer + - "null" + required: + - id + - updatedAt + forms: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + afterFormContent: + type: + - string + - "null" + beforeFormContent: + type: + - string + - "null" + campaignId: + type: + - integer + - "null" + checkboxAlignment: + type: + - string + - "null" + createdAt: + type: + - string + - "null" + format: date-time + createdById: + type: + - integer + - "null" + embedCode: + type: + - string + - "null" + folderId: + type: + - integer + - "null" + fontColor: + type: + - string + - "null" + fontFamily: + type: + - string + - "null" + fontSize: + type: + - string + - "null" + id: + type: integer + isAlwaysDisplay: + type: + - boolean + - "null" + isCaptchaEnabled: + type: + - boolean + - "null" + isCookieless: + type: + - boolean + - "null" + isDeleted: + type: + - boolean + - "null" + isUseRedirectLocation: + type: + - boolean + - "null" + labelAlignment: + type: + - string + - "null" + layoutTemplateId: + type: + - integer + - "null" + name: + type: + - string + - "null" + radioAlignment: + type: + - string + - "null" + redirectLocation: + type: + - string + - "null" + requiredCharacter: + type: + - string + - "null" + salesforceId: + type: + - string + - "null" + showNotProspect: + type: + - boolean + - "null" + submitButtonText: + type: + - string + - "null" + thankYouCode: + type: + - string + - "null" + thankYouContent: + type: + - string + - "null" + updatedAt: + type: + - string + - "null" + format: date-time + updatedById: + type: + - integer + - "null" + required: + - id + form_fields: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + createdAt: + type: + - string + - "null" + format: date-time + createdById: + type: + - integer + - "null" + cssClasses: + type: + - string + - "null" + dataFormat: + type: + - string + - "null" + errorMessage: + type: + - string + - "null" + formId: + type: + - integer + - "null" + hasDependents: + type: + - boolean + - "null" + hasProgressives: + type: + - boolean + - "null" + hasValues: + type: + - boolean + - "null" + id: + type: integer + isAlwaysDisplay: + type: + - boolean + - "null" + isDoNotPrefill: + type: + - boolean + - "null" + isMaintainInitialValue: + type: + - boolean + - "null" + isRequired: + type: + - boolean + - "null" + label: + type: + - string + - "null" + prospectApiFieldId: + type: + - string + - "null" + sortOrder: + type: + - integer + - "null" + updatedAt: + type: + - string + - "null" + format: date-time + updatedById: + type: + - integer + - "null" + required: + - id + - updatedAt + form_handlers: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + campaignId: + type: + - integer + - "null" + createdAt: + type: + - string + - "null" + format: date-time + createdById: + type: + - integer + - "null" + embedCode: + type: + - string + - "null" + errorLocation: + type: + - string + - "null" + folderId: + type: + - integer + - "null" + id: + type: integer + isAlwaysEmail: + type: + - boolean + - "null" + isCookieless: + type: + - boolean + - "null" + isDataForwarded: + type: + - boolean + - "null" + isDeleted: + type: + - boolean + - "null" + name: + type: + - string + - "null" + salesforceId: + type: + - string + - "null" + successLocation: + type: + - string + - "null" + trackerDomainId: + type: + - integer + - "null" + updatedById: + type: + - integer + - "null" + required: + - id + form_handler_fields: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + createdAt: + type: + - string + - "null" + format: date-time + createdById: + type: + - integer + - "null" + dataFormat: + type: + - string + - "null" + errorMessage: + type: + - string + - "null" + formHandlerId: + type: + - integer + - "null" + id: + type: integer + isMaintainInitialValue: + type: + - boolean + - "null" + isRequired: + type: + - boolean + - "null" + name: + type: + - string + - "null" + prospectApiFieldId: + type: + - string + - "null" + required: + - id + landing_pages: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + description: + type: + - string + - "null" + archiveDate: + type: + - string + - "null" + format: date + bitlyIsPersonalized: + type: + - boolean + - "null" + bitlyShortUrl: + type: + - string + - "null" + campaignId: + type: + - integer + - "null" + createdAt: + type: + - string + - "null" + format: date-time + createdById: + type: + - integer + - "null" + folderId: + type: + - integer + - "null" + formId: + type: + - integer + - "null" + id: + type: integer + isDeleted: + type: + - boolean + - "null" + isDoNotIndex: + type: + - boolean + - "null" + isUseRedirectLocation: + type: + - boolean + - "null" + layoutTableBorder: + type: + - integer + - "null" + layoutTemplateId: + type: + - integer + - "null" + layoutType: + type: + - string + - "null" + name: + type: + - string + - "null" + redirectLocation: + type: + - string + - "null" + salesforceId: + type: + - string + - "null" + title: + type: + - string + - "null" + trackerDomainId: + type: + - integer + - "null" + updatedAt: + type: + - string + - "null" + format: date-time + updatedById: + type: + - integer + - "null" + url: + type: + - string + - "null" + vanityUrl: + type: + - string + - "null" + vanityUrlPath: + type: + - string + - "null" + required: + - id + - updatedAt + layout_templates: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + createdAt: + type: + - string + - "null" + format: date-time + createdById: + type: + - integer + - "null" + folderId: + type: + - integer + - "null" + formContent: + type: + - string + - "null" + id: + type: integer + isDeleted: + type: + - boolean + - "null" + isIncludeDefaultCss: + type: + - boolean + - "null" + layoutContent: + type: + - string + - "null" + name: + type: + - string + - "null" + siteSearchContent: + type: + - string + - "null" + updatedAt: + type: + - string + - "null" + format: date-time + updatedById: + type: + - integer + - "null" + required: + - id + lifecycle_stages: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + createdAt: + type: + - string + - "null" + format: date-time + id: + type: integer + isDeleted: + type: + - boolean + - "null" + isLocked: + type: + - boolean + - "null" + matchType: + type: + - string + - "null" + name: + type: + - string + - "null" + position: + type: + - integer + - "null" + updatedAt: + type: + - string + - "null" + format: date-time + required: + - id + - updatedAt + lifecycle_histories: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + createdAt: + type: + - string + - "null" + format: date-time + id: + type: integer + nextStageId: + type: + - integer + - "null" + previousStageId: + type: + - integer + - "null" + prospectId: + type: + - integer + - "null" + secondsElapsed: + type: + - integer + - "null" + required: + - id + - createdAt + list_emails: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + campaignId: + type: + - integer + - "null" + clientType: + type: + - string + - "null" + createdAt: + type: + - string + - "null" + format: date-time + createdById: + type: + - integer + - "null" + emailTemplateId: + type: + - integer + - "null" + folderId: + type: + - integer + - "null" + id: + type: integer + isDeleted: + type: + - boolean + - "null" + isOperational: + type: + - boolean + - "null" + isPaused: + type: + - boolean + - "null" + isSent: + type: + - boolean + - "null" + name: + type: + - string + - "null" + replyToOptions: + type: + - array + - "null" + items: + type: + - object + - "null" + properties: + type: + type: + - string + - "null" + accountCustomFieldId: + type: + - integer + - "null" + address: + type: + - string + - "null" + prospectCustomFieldId: + type: + - integer + - "null" + userId: + type: + - integer + - "null" + senderOptions: + type: + - array + - "null" + items: + type: + - object + - "null" + properties: + type: + type: + - string + - "null" + accountCustomFieldId: + type: + - integer + - "null" + address: + type: + - string + - "null" + name: + type: + - string + - "null" + prospectCustomFieldId: + type: + - integer + - "null" + userId: + type: + - integer + - "null" + sentAt: + type: + - string + - "null" + subject: + type: + - string + - "null" + trackerDomainId: + type: + - integer + - "null" + updatedAt: + type: + - string + - "null" + format: date-time + updatedById: + type: + - integer + - "null" + required: + - id + - updatedAt + opportunities: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + campaignId: + type: + - integer + - "null" + closedAt: + type: + - string + - "null" + format: date-time + createdAt: + type: + - string + - "null" + format: date-time + createdById: + type: + - integer + - "null" + id: + type: integer + name: + type: + - string + - "null" + probability: + type: + - integer + - "null" + salesforceId: + type: + - string + - "null" + stage: + type: + - string + - "null" + status: + type: + - string + - "null" + updatedAt: + type: + - string + - "null" + format: date-time + updatedById: + type: + - integer + - "null" + value: + type: + - number + - "null" + required: + - id + - updatedAt + tags: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + createdAt: + type: + - string + - "null" + format: date-time + createdById: + type: + - integer + - "null" + id: + type: integer + name: + type: + - string + - "null" + objectCount: + type: + - integer + - "null" + updatedAt: + type: + - string + - "null" + format: date-time + updatedById: + type: + - integer + - "null" + required: + - id + - updatedAt + tracker_domains: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + createdAt: + type: + - string + - "null" + format: date-time + createdById: + type: + - integer + - "null" + defaultCampaignId: + type: + - integer + - "null" + domain: + type: + - string + - "null" + httpsStatus: + type: + - string + - "null" + id: + type: integer + isDeleted: + type: + - boolean + - "null" + isPrimary: + type: + - boolean + - "null" + sslRequestedById: + type: + - integer + - "null" + sslStatus: + type: + - string + - "null" + sslStatusDetails: + type: + - string + - "null" + trackingCode: + type: + - string + - "null" + updatedAt: + type: + - string + - "null" + format: date-time + updatedById: + type: + - integer + - "null" + validatedAt: + type: + - string + - "null" + format: date-time + validationStatus: + type: + - string + - "null" + vanityUrlStatus: + type: + - string + - "null" + required: + - id + visitor_page_views: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + campaignId: + type: + - integer + - "null" + createdAt: + type: + - string + - "null" + format: date-time + durationInSeconds: + type: + - integer + - "null" + id: + type: integer + salesforceId: + type: + - string + - "null" + title: + type: + - string + - "null" + url: + type: + - string + - "null" + visitId: + type: + - integer + - "null" + visitorId: + type: + - integer + - "null" + required: + - id + - createdAt + account: + type: object + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + properties: + addressOne: + type: + - string + - "null" + addressTwo: + type: + - string + - "null" + adminId: + type: + - integer + - "null" + apiCallsUsed: + type: + - integer + - "null" + city: + type: + - string + - "null" + company: + type: + - string + - "null" + country: + type: + - string + - "null" + createdAt: + type: + - string + - "null" + format: date-time + createdById: + type: + - integer + - "null" + fax: + type: + - string + - "null" + id: + type: integer + level: + type: + - string + - "null" + maximumDailyApiCalls: + type: + - integer + - "null" + phone: + type: + - string + - "null" + pluginCampaignId: + type: + - integer + - "null" + state: + type: + - string + - "null" + territory: + type: + - string + - "null" + updatedAt: + type: + - string + - "null" + format: date-time + updatedById: + type: + - integer + - "null" + website: + type: + - string + - "null" + zip: + type: + - string + - "null" + required: + - id + - createdAt + - updatedAt + custom_fields: + type: object + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + createdAt: + type: + - string + - "null" + format: date-time + createdById: + type: + - integer + - "null" + fieldId: + type: + - integer + - "null" + id: + type: integer + isAnalyticsSynced: + type: + - boolean + - "null" + isRecordMultipleResponses: + type: + - boolean + - "null" + isRequired: + type: + - boolean + - "null" + isUseValues: + type: + - boolean + - "null" + name: + type: + - string + - "null" + salesforceId: + type: + - string + - "null" + updatedAt: + type: + - string + - "null" + format: date-time + updatedById: + type: + - integer + - "null" + required: + - id + - createdAt + - updatedAt + dynamic_contents: + type: object + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + properties: + basedOn: + type: + - string + - "null" + basedOnProspectApiFieldId: + type: + - integer + - "null" + createdAt: + type: + - string + - "null" + format: date-time + createdById: + type: + - integer + - "null" + embedCode: + type: + - string + - "null" + embedUrl: + type: + - string + - "null" + folderId: + type: + - integer + - "null" + id: + type: integer + isDeleted: + type: + - boolean + - "null" + name: + type: + - string + - "null" + tagReplacementLanguage: + type: + - string + - "null" + trackerDomainId: + type: + - integer + - "null" + updatedAt: + type: + - string + - "null" + format: date-time + updatedById: + type: + - integer + - "null" + required: + - id + - updatedAt + dynamic_content_variations: + type: object + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + properties: + id: + type: integer + dynamicContentId: + type: + - integer + - "null" + comparison: + type: + - string + - "null" + operator: + type: + - string + - "null" + value1: + type: + - string + - "null" + value2: + type: + - string + - "null" + content: + type: + - string + - "null" + required: + - id diff --git a/airbyte-integrations/connectors/source-pardot/metadata.yaml b/airbyte-integrations/connectors/source-pardot/metadata.yaml index 55e8e0bfbf5f..1596dd7938af 100644 --- a/airbyte-integrations/connectors/source-pardot/metadata.yaml +++ b/airbyte-integrations/connectors/source-pardot/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: ad15c7ba-72a7-440b-af15-b9a963dc1a8a - dockerImageTag: 0.2.0 + dockerImageTag: 1.0.0 dockerRepository: airbyte/source-pardot githubIssueLabel: source-pardot icon: salesforcepardot.svg @@ -18,6 +18,11 @@ data: oss: enabled: true releaseStage: alpha + releases: + breakingChanges: + 1.0.0: + message: Most streams have been migrated to use Pardot API V5 in this release. The authentication flow, which was previously broken, should now work for new connections using this version. + upgradeDeadline: "2024-12-26" documentationUrl: https://docs.airbyte.com/integrations/sources/pardot tags: - language:manifest-only @@ -29,5 +34,5 @@ data: connectorTestSuitesOptions: - suite: unitTests connectorBuildOptions: - baseImage: docker.io/airbyte/source-declarative-manifest:5.13.0@sha256:ffc5977f59e1f38bf3f5dd70b6fa0520c2450ebf85153c5a8df315b8c918d5c3 + baseImage: docker.io/airbyte/source-declarative-manifest:6.10.0@sha256:58722e84dbd06bb2af9250e37d24d1c448e247fc3a84d75ee4407d52771b6f03 metadataSpecVersion: "1.0" diff --git a/docs/integrations/sources/pardot-migrations.md b/docs/integrations/sources/pardot-migrations.md new file mode 100644 index 000000000000..a6dfcef4abaa --- /dev/null +++ b/docs/integrations/sources/pardot-migrations.md @@ -0,0 +1,14 @@ +# Pardot Migration Guide + +## Upgrading to 1.0.0 + +Version 1.0.0 contains a number of fixes and updates to the Pardot source connector: + +- Fixed authentication +- Migrate all existing streams to Pardot v5 API (except email_clicks which is only available in v4) +- Re-implement incremental syncs for existing streams where possible +- Add 23 new streams from the v5 API (folders, emails, engagement_studio_programs, folder_contents, forms, form_fields, form_handlers, form_handler_fields, landing_pages, layout_templates, lifecycle_stages, lifecycle_histories, list_emails, opportunities, tags, tracker_domains, visitor_page_views) +- Add additional configuration options to better handle large accounts (e.g. adjustable split-up windows, page size) +- Align to Pardot-recommended sort/filter/pagination conventions to avoid timeouts (based on Pardot support case #469072278) + +The previous implementation of the authentication flow was no longer functional, preventing the instantiation of new sources. All users with existing connections should reconfigure the source and go through the authentication flow before attempting to sync with this connector. OSS users should be sure to manually update their source version to >=1.0.0 before attempting to configure this source. diff --git a/docs/integrations/sources/pardot.md b/docs/integrations/sources/pardot.md index f9907d617348..5c90ca581663 100644 --- a/docs/integrations/sources/pardot.md +++ b/docs/integrations/sources/pardot.md @@ -1,66 +1,101 @@ -# Pardot +# Pardot (Salesforce Marketing Cloud Account Engagement) ## Overview -The Airbyte Source for [Salesforce Pardot](https://www.pardot.com/) +This page contains the setup guide and reference information for the [Pardot (Salesforce Marketing Cloud Account Engagement)](https://www.salesforce.com/marketing/b2b-automation/) source connector. -The Pardot supports full refresh syncs +## Prerequisites -### Output schema +- Pardot/Marketing Cloud Account Engagement account +- Pardot Business Unit ID +- Client ID +- Client Secret +- Refresh Token -Several output streams are available from this source: +## Setup Guide -- [Campaigns](https://developer.salesforce.com/docs/marketing/pardot/guide/campaigns-v4.html) -- [EmailClicks](https://developer.salesforce.com/docs/marketing/pardot/guide/batch-email-clicks-v4.html) -- [ListMembership](https://developer.salesforce.com/docs/marketing/pardot/guide/list-memberships-v4.html) -- [Lists](https://developer.salesforce.com/docs/marketing/pardot/guide/lists-v4.html) -- [ProspectAccounts](https://developer.salesforce.com/docs/marketing/pardot/guide/prospect-accounts-v4.html) -- [Prospects](https://developer.salesforce.com/docs/marketing/pardot/guide/prospects-v4.html) -- [Users](https://developer.salesforce.com/docs/marketing/pardot/guide/users-v4.html) -- [VisitorActivities](https://developer.salesforce.com/docs/marketing/pardot/guide/visitor-activities-v4.html) -- [Visitors](https://developer.salesforce.com/docs/marketing/pardot/guide/visitors-v4.html) -- [Visits](https://developer.salesforce.com/docs/marketing/pardot/guide/visits-v4.html) +### Required configuration options +- **Pardot Business Unit ID** (`pardot_business_unit_id`): This value uniquely identifies your account, and can be found at Setup > Pardot > Pardot Account Setup -If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose) +- **Client ID** (`client_id`): The Consumer Key that can be found when viewing your app in Salesforce -### Features +- **Client Secret** (`client_secret`): The Consumer Secret that can be found when viewing your app in Salesforce -| Feature | Supported? | -| :---------------- | :--------- | -| Full Refresh Sync | Yes | -| Incremental Sync | No | -| SSL connection | No | -| Namespaces | No | +- **Refresh Token** (`refresh_token`): Salesforce Refresh Token used for Airbyte to access your Salesforce account. If you don't know what this is, follow [this guide](https://medium.com/@bpmmendis94/obtain-access-refresh-tokens-from-salesforce-rest-api-a324fe4ccd9b) to retrieve it. -### Performance considerations +### Optional configuration options +- **Start Date** (`start_date`): UTC date and time in the format `2020-01-25T00:00:00Z`. Any data before this date will not be replicated. Defaults to `2007-01-01T00:00:00Z` (the year Pardot was launched) -The Pardot connector should not run into Pardot API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. +- **Page Size Limit** (`page_size`): The default page size to return; defaults to `1000` (which is Pardot's maximum). Does not apply to the Email Clicks stream which uses the v4 API and is limited to 200 per page. -## Getting started +- **Default Split Up Interval** (`split_up_interval`): The default split up interval is used on incremental streams to prevent hitting Pardots limit of 100 pages per result (effectively 100K records on most endpoints). The default is `P3M`, which will break incremental requests into three-month groupings. If you expect more than 100K records to be modified between syncs in a single endpoint, you can increase the granularity to `P1M`, `P14D`, `P7D`, `P3D`, or `P1D` to reduce the maximum records to be paged through per split. For small accounts unlikely to hit the limit, decreasing the granularity to `P6M` or `P1Y` may increase the speed of those syncs, especially the initial backfill. -### Requirements +- **Is Sandbox App?** (`is_sandbox`): Whether or not the app is in a Salesforce sandbox. If you do not know what this is, assume it is false. -- Pardot Account -- Pardot Business Unit ID -- Client ID -- Client Secret -- Refresh Token -- Start Date -- Is Sandbox environment? +## Supported Sync Modes + +The Pardot source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-modes): + +- Full Refresh +- Incremental -### Setup guide +Incremental streams are based on the Pardot API's `UpdatedAt` field when the object is updateable and the API supports it; otherwise `CreatedAt` or `Id` are used in that order of preference. + +### Performance Considerations + +The Pardot connector should not run into Pardot API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. -- `pardot_business_unit_id`: Pardot Business ID, can be found at Setup > Pardot > Pardot Account Setup -- `client_id`: The Consumer Key that can be found when viewing your app in Salesforce -- `client_secret`: The Consumer Secret that can be found when viewing your app in Salesforce -- `refresh_token`: Salesforce Refresh Token used for Airbyte to access your Salesforce account. If you don't know what this is, follow [this guide](https://medium.com/@bpmmendis94/obtain-access-refresh-tokens-from-salesforce-rest-api-a324fe4ccd9b) to retrieve it. -- `start_date`: UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. Leave blank to skip this filter -- `is_sandbox`: Whether or not the app is in a Salesforce sandbox. If you do not know what this is, assume it is false. +:::tip + +Due to timeouts on large accounts, the Split Up Interval (the time period used to page through incrementally) is surfaced as a configuration option. While the default should work for most accounts, large accounts may need to increase the granularity from the default. Similarly, small accounts may see faster initial syncs with a longer interval. See the Optional Configuration Options section for more details. + +::: + +## Supported Streams + +Several output streams are available from this source. Unless noted otherwise, streams are from Pardot's v5 API: + +- [Account (Metadata)](https://developer.salesforce.com/docs/marketing/pardot/guide/account-v5.html) (full refresh) +- [Campaigns](https://developer.salesforce.com/docs/marketing/pardot/guide/campaign-v5.html) (incremental) +- [Custom Fields](https://developer.salesforce.com/docs/marketing/pardot/guide/custom-field-v5.html) (incremental) +- [Custom Redirects](https://developer.salesforce.com/docs/marketing/pardot/guide/custom-redirect-v5.html) (full refresh) +- [Dynamic Content](https://developer.salesforce.com/docs/marketing/pardot/guide/dynamic-content-v5.html) (incremental) +- [Dynamic Content Variations](https://developer.salesforce.com/docs/marketing/pardot/guide/dynamic-content-variation.html) (incremental parent) +- [Emails](https://developer.salesforce.com/docs/marketing/pardot/guide/email-v5.html) (incremental) +- [Email Clicks (v4 API)](https://developer.salesforce.com/docs/marketing/pardot/guide/batch-email-clicks-v4.html) (incremental) +- [Engagement Studio Programs](https://developer.salesforce.com/docs/marketing/pardot/guide/engagement-studio-program-v5.html) (incremental) +- [Files](https://developer.salesforce.com/docs/marketing/pardot/guide/export-v5.html) (full refresh) +- [Folders](https://developer.salesforce.com/docs/marketing/pardot/guide/folder-v5.html) (full refresh) +- [Folder Contents](https://developer.salesforce.com/docs/marketing/pardot/guide/folder-contents-v5.html) (incremental) +- [Forms](https://developer.salesforce.com/docs/marketing/pardot/guide/form-v5.html) (full refresh) +- [Form Fields](https://developer.salesforce.com/docs/marketing/pardot/guide/form-field-v5.html) (incremental) +- [Form Handlers](https://developer.salesforce.com/docs/marketing/pardot/guide/form-handler-v5.html) (full refresh) +- [Form Handler Fields](https://developer.salesforce.com/docs/marketing/pardot/guide/form-handler-field-v5.html) (full refresh) +- [Landing Pages](https://developer.salesforce.com/docs/marketing/pardot/guide/landing-page-v5.html) (incremental) +- [Layout Templates](https://developer.salesforce.com/docs/marketing/pardot/guide/layout-template-v5.html) (full refresh) +- [Lifecycle Stages](https://developer.salesforce.com/docs/marketing/pardot/guide/lifecycle-stage-v5.html) (incremental) +- [Lifecycle Histories](https://developer.salesforce.com/docs/marketing/pardot/guide/lifecycle-history-v5.html) (incremental) +- [Lists](https://developer.salesforce.com/docs/marketing/pardot/guide/list-v5.html) (incremental) +- [List Emails](https://developer.salesforce.com/docs/marketing/pardot/guide/list-email-v5.html) (incremental) +- [List Memberships](https://developer.salesforce.com/docs/marketing/pardot/guide/list-membership-v5.html) (incremental) +- [Opportunities](https://developer.salesforce.com/docs/marketing/pardot/guide/opportunity-v5.html) (incremental) +- [Prospects](https://developer.salesforce.com/docs/marketing/pardot/guide/prospect-v5.html) (incremental) +- [Prospect Accounts](https://developer.salesforce.com/docs/marketing/pardot/guide/prospect-account-v5.html) (full refresh) +- [Tags](https://developer.salesforce.com/docs/marketing/pardot/guide/tag-v5.html) (incremental) +- [Tracker Domains](https://developer.salesforce.com/docs/marketing/pardot/guide/tracker-domain-v5.html) (full refresh) +- [Users](https://developer.salesforce.com/docs/marketing/pardot/guide/user-v5.html) (incremental) +- [Visitors](https://developer.salesforce.com/docs/marketing/pardot/guide/visitor-v5.html) (incremental) +- [Visitor Activity](https://developer.salesforce.com/docs/marketing/pardot/guide/visitor-activity-v5.html) (incremental) +- [Visitor Page Views](https://developer.salesforce.com/docs/marketing/pardot/guide/visitor-page-view-v5.html) (incremental) +- [Visits](https://developer.salesforce.com/docs/marketing/pardot/guide/visit-v5.html) (incremental) + +If there are more endpoints you'd like Airbyte to support, please [create an issue](https://github.com/airbytehq/airbyte/issues/new/choose). ## Changelog | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------- | +| 1.0.0 | 2023-12-12 | [49424](https://github.com/airbytehq/airbyte/pull/49424) | Update streams to API V5. Fix auth flow | | 0.2.0 | 2024-10-13 | [44528](https://github.com/airbytehq/airbyte/pull/44528) | Migrate to LowCode then Manifest-only | | 0.1.22 | 2024-10-12 | [46778](https://github.com/airbytehq/airbyte/pull/46778) | Update dependencies | | 0.1.21 | 2024-10-05 | [46441](https://github.com/airbytehq/airbyte/pull/46441) | Update dependencies | From 6341ead1979259ddd23c83c39b78f4a7b44af2cc Mon Sep 17 00:00:00 2001 From: Yarden Carmeli <74664736+yardencarmeli@users.noreply.github.com> Date: Tue, 17 Dec 2024 09:18:28 -0800 Subject: [PATCH 4/7] Updating CDC sources documentation to clarify incremental sync method limitation. (#49823) --- docs/integrations/sources/mssql.md | 4 +++- docs/integrations/sources/mysql/mysql-troubleshooting.md | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md index 5e1ceaa3778e..689e83f07c22 100644 --- a/docs/integrations/sources/mssql.md +++ b/docs/integrations/sources/mssql.md @@ -101,7 +101,9 @@ approaches CDC. - The SQL Server CDC feature processes changes that occur in user-created tables only. You cannot enable CDC on the SQL Server master database. - Using variables with partition switching on databases or tables with change data capture \(CDC\) - is not supported for the `ALTER TABLE` ... `SWITCH TO` ... `PARTITION` ... statement + is not supported for the `ALTER TABLE` ... `SWITCH TO` ... `PARTITION` ... statement. +- CDC incremental syncing is only available for tables with at least one primary key. Tables without primary keys can still be replicated by CDC but only in Full Refresh mode. + For more information on CDC limitations, refer to our [CDC Limitations doc](https://docs.airbyte.com/understanding-airbyte/cdc#limitations). - Our CDC implementation uses at least once delivery for all change records. - Read more on CDC limitations in the [Microsoft docs](https://docs.microsoft.com/en-us/sql/relational-databases/track-changes/about-change-data-capture-sql-server?view=sql-server-2017#limitations). diff --git a/docs/integrations/sources/mysql/mysql-troubleshooting.md b/docs/integrations/sources/mysql/mysql-troubleshooting.md index b733ef331181..84e403f91323 100644 --- a/docs/integrations/sources/mysql/mysql-troubleshooting.md +++ b/docs/integrations/sources/mysql/mysql-troubleshooting.md @@ -10,6 +10,8 @@ - Make sure to read our [CDC docs](../../../understanding-airbyte/cdc.md) to see limitations that impact all databases using CDC replication. - Our CDC implementation uses at least once delivery for all change records. +- To enable CDC with incremental sync, ensure the table has at least one primary key. + Tables without primary keys can still be replicated by CDC but only in Full Refresh mode. ### Vendor-Specific Connector Limitations From 3f4e4d6c5b9a6da372b446aea826badb5cc3cb6d Mon Sep 17 00:00:00 2001 From: Augustin <augustin@airbyte.io> Date: Tue, 17 Dec 2024 18:30:15 +0100 Subject: [PATCH 5/7] base-images: release a base image for our java connectors (#49831) --- airbyte-ci/connectors/base_images/README.md | 32 +++++- .../base_images/base_images/java/__init__.py | 3 + .../base_images/base_images/java/bases.py | 102 ++++++++++++++++++ .../base_images/base_images/root_images.py | 7 ++ .../base_images/base_images/sanity_checks.py | 16 +++ .../base_images/templates/README.md.j2 | 8 +- .../base_images/base_images/utils/dagger.py | 6 ++ .../base_images/version_registry.py | 14 ++- .../airbyte_java_connector_base.json | 22 ++++ .../connectors/base_images/pyproject.toml | 2 +- 10 files changed, 206 insertions(+), 6 deletions(-) create mode 100644 airbyte-ci/connectors/base_images/base_images/java/__init__.py create mode 100644 airbyte-ci/connectors/base_images/base_images/java/bases.py create mode 100644 airbyte-ci/connectors/base_images/base_images/utils/dagger.py create mode 100644 airbyte-ci/connectors/base_images/generated/changelogs/airbyte_java_connector_base.json diff --git a/airbyte-ci/connectors/base_images/README.md b/airbyte-ci/connectors/base_images/README.md index fbe05942d497..8b6bf9b40237 100644 --- a/airbyte-ci/connectors/base_images/README.md +++ b/airbyte-ci/connectors/base_images/README.md @@ -6,7 +6,7 @@ Our connector build pipeline ([`airbyte-ci`](https://github.com/airbytehq/airbyt Our base images are declared in code, using the [Dagger Python SDK](https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/). - [Python base image code declaration](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/base_images/base_images/python/bases.py) -- ~Java base image code declaration~ *TODO* +- [Java base image code declaration](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/base_images/base_images/java/bases.py) ## Where are the Dockerfiles? @@ -39,6 +39,20 @@ RUN mkdir -p 755 /usr/share/nltk_data +### Example for `airbyte/java-connector-base`: +```dockerfile +FROM docker.io/amazoncorretto:21-al2023@sha256:5454cb606e803fce56861fdbc9eab365eaa2ab4f357ceb8c1d56f4f8c8a7bc33 +RUN sh -c set -o xtrace && yum update -y --security && yum install -y tar openssl findutils && yum clean all +ENV AIRBYTE_SPEC_CMD=/airbyte/javabase.sh --spec +ENV AIRBYTE_CHECK_CMD=/airbyte/javabase.sh --check +ENV AIRBYTE_DISCOVER_CMD=/airbyte/javabase.sh --discover +ENV AIRBYTE_READ_CMD=/airbyte/javabase.sh --read +ENV AIRBYTE_WRITE_CMD=/airbyte/javabase.sh --write +ENV AIRBYTE_ENTRYPOINT=/airbyte/base.sh +``` + + + ## Base images @@ -59,6 +73,17 @@ RUN mkdir -p 755 /usr/share/nltk_data | 1.0.0 | ✅| docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27 | Initial release: based on Python 3.9.18, on slim-bookworm system, with pip==23.2.1 and poetry==1.6.1 | +### `airbyte/java-connector-base` + +| Version | Published | Docker Image Address | Changelog | +|---------|-----------|--------------|-----------| +| 1.0.0 | ✅| docker.io/airbyte/java-connector-base:1.0.0@sha256:be86e5684e1e6d9280512d3d8071b47153698fe08ad990949c8eeff02803201a | Create a base image for our java connectors based on Amazon Corretto. | +| 1.0.0-rc.4 | ✅| docker.io/airbyte/java-connector-base:1.0.0-rc.4@sha256:be86e5684e1e6d9280512d3d8071b47153698fe08ad990949c8eeff02803201a | Bundle yum calls in a single RUN | +| 1.0.0-rc.3 | ✅| docker.io/airbyte/java-connector-base:1.0.0-rc.3@sha256:be86e5684e1e6d9280512d3d8071b47153698fe08ad990949c8eeff02803201a | | +| 1.0.0-rc.2 | ✅| docker.io/airbyte/java-connector-base:1.0.0-rc.2@sha256:fca66e81b4d2e4869a03b57b1b34beb048e74f5d08deb2046c3bb9919e7e2273 | Set entrypoint to base.sh | +| 1.0.0-rc.1 | ✅| docker.io/airbyte/java-connector-base:1.0.0-rc.1@sha256:886a7ce7eccfe3c8fb303511d0e46b83b7edb4f28e3705818c090185ba511fe7 | Create a base image for our java connectors. | + + ## How to release a new base image version (example for Python) ### Requirements @@ -102,6 +127,9 @@ poetry run mypy base_images --check-untyped-defs ## CHANGELOG +### 1.4.0 +- Declare a base image for our java connectors. + ### 1.3.1 - Update the crane image address. The previous address was deleted by the maintainer. @@ -120,4 +148,4 @@ poetry run mypy base_images --check-untyped-defs ### 1.0.1 -- Bumped dependencies ([#42581](https://github.com/airbytehq/airbyte/pull/42581)) +- Bumped dependencies ([#42581](https://github.com/airbytehq/airbyte/pull/42581)) \ No newline at end of file diff --git a/airbyte-ci/connectors/base_images/base_images/java/__init__.py b/airbyte-ci/connectors/base_images/base_images/java/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/base_images/base_images/java/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/base_images/base_images/java/bases.py b/airbyte-ci/connectors/base_images/base_images/java/bases.py new file mode 100644 index 000000000000..ed820e5b9863 --- /dev/null +++ b/airbyte-ci/connectors/base_images/base_images/java/bases.py @@ -0,0 +1,102 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from __future__ import annotations + +from typing import Callable, Final + +import dagger +from base_images import bases, published_image +from base_images import sanity_checks as base_sanity_checks +from base_images.python import sanity_checks as python_sanity_checks +from base_images.root_images import AMAZON_CORRETTO_21_AL_2023 +from base_images.utils.dagger import sh_dash_c + + +class AirbyteJavaConnectorBaseImage(bases.AirbyteConnectorBaseImage): + # TODO: remove this once we want to build the base image with the airbyte user. + USER: Final[str] = "root" + + root_image: Final[published_image.PublishedImage] = AMAZON_CORRETTO_21_AL_2023 + repository: Final[str] = "airbyte/java-connector-base" + + DD_AGENT_JAR_URL: Final[str] = "https://dtdg.co/latest-java-tracer" + BASE_SCRIPT_URL = "https://raw.githubusercontent.com/airbytehq/airbyte/6d8a3a2bc4f4ca79f10164447a90fdce5c9ad6f9/airbyte-integrations/bases/base/base.sh" + JAVA_BASE_SCRIPT_URL: Final[ + str + ] = "https://raw.githubusercontent.com/airbytehq/airbyte/6d8a3a2bc4f4ca79f10164447a90fdce5c9ad6f9/airbyte-integrations/bases/base-java/javabase.sh" + + def get_container(self, platform: dagger.Platform) -> dagger.Container: + """Returns the container used to build the base image for java connectors + We currently use the Amazon coretto image as a base. + We install some packages required to build java connectors. + We also download the datadog java agent jar and the javabase.sh script. + We set some env variables used by the javabase.sh script. + + Args: + platform (dagger.Platform): The platform this container should be built for. + + Returns: + dagger.Container: The container used to build the base image. + """ + + return ( + # TODO: Call this when we want to build the base image with the airbyte user + # self.get_base_container(platform) + self.dagger_client.container(platform=platform) + .from_(self.root_image.address) + # Bundle RUN commands together to reduce the number of layers. + .with_exec( + sh_dash_c( + [ + # Update first, but in the same .with_exec step as the package installation. + # Otherwise, we risk caching stale package URLs. + "yum update -y --security", + # tar is equired to untar java connector binary distributions. + # openssl is required because we need to ssh and scp sometimes. + # findutils is required for xargs, which is shipped as part of findutils. + f"yum install -y tar openssl findutils", + # Remove any dangly bits. + "yum clean all", + ] + ) + ) + .with_workdir("/airbyte") + # Copy the datadog java agent jar from the internet. + .with_file("dd-java-agent.jar", self.dagger_client.http(self.DD_AGENT_JAR_URL)) + # Copy base.sh from the git repo. + .with_file("base.sh", self.dagger_client.http(self.BASE_SCRIPT_URL)) + # Copy javabase.sh from the git repo. + .with_file("javabase.sh", self.dagger_client.http(self.JAVA_BASE_SCRIPT_URL)) + # Set a bunch of env variables used by base.sh. + .with_env_variable("AIRBYTE_SPEC_CMD", "/airbyte/javabase.sh --spec") + .with_env_variable("AIRBYTE_CHECK_CMD", "/airbyte/javabase.sh --check") + .with_env_variable("AIRBYTE_DISCOVER_CMD", "/airbyte/javabase.sh --discover") + .with_env_variable("AIRBYTE_READ_CMD", "/airbyte/javabase.sh --read") + .with_env_variable("AIRBYTE_WRITE_CMD", "/airbyte/javabase.sh --write") + .with_env_variable("AIRBYTE_ENTRYPOINT", "/airbyte/base.sh") + .with_entrypoint(["/airbyte/base.sh"]) + ) + + async def run_sanity_checks(self, platform: dagger.Platform): + """Runs sanity checks on the base image container. + This method is called before image publication. + Consider it like a pre-flight check before take-off to the remote registry. + + Args: + platform (dagger.Platform): The platform on which the sanity checks should run. + """ + container = self.get_container(platform) + await base_sanity_checks.check_user_can_read_dir(container, self.USER, self.AIRBYTE_DIR_PATH) + await base_sanity_checks.check_user_can_write_dir(container, self.USER, self.AIRBYTE_DIR_PATH) + await base_sanity_checks.check_file_exists(container, "/airbyte/dd-java-agent.jar") + await base_sanity_checks.check_file_exists(container, "/airbyte/base.sh") + await base_sanity_checks.check_file_exists(container, "/airbyte/javabase.sh") + await base_sanity_checks.check_env_var_with_printenv(container, "AIRBYTE_SPEC_CMD", "/airbyte/javabase.sh --spec") + await base_sanity_checks.check_env_var_with_printenv(container, "AIRBYTE_CHECK_CMD", "/airbyte/javabase.sh --check") + await base_sanity_checks.check_env_var_with_printenv(container, "AIRBYTE_DISCOVER_CMD", "/airbyte/javabase.sh --discover") + await base_sanity_checks.check_env_var_with_printenv(container, "AIRBYTE_READ_CMD", "/airbyte/javabase.sh --read") + await base_sanity_checks.check_env_var_with_printenv(container, "AIRBYTE_WRITE_CMD", "/airbyte/javabase.sh --write") + await base_sanity_checks.check_env_var_with_printenv(container, "AIRBYTE_ENTRYPOINT", "/airbyte/base.sh") + await base_sanity_checks.check_a_command_is_available_using_version_option(container, "tar") + await base_sanity_checks.check_a_command_is_available_using_version_option(container, "openssl", "version") diff --git a/airbyte-ci/connectors/base_images/base_images/root_images.py b/airbyte-ci/connectors/base_images/base_images/root_images.py index 8cb7036d22ef..dcd0892a8f6c 100644 --- a/airbyte-ci/connectors/base_images/base_images/root_images.py +++ b/airbyte-ci/connectors/base_images/base_images/root_images.py @@ -24,3 +24,10 @@ tag="3.10.14-slim-bookworm", sha="2407c61b1a18067393fecd8a22cf6fceede893b6aaca817bf9fbfe65e33614a3", ) + +AMAZON_CORRETTO_21_AL_2023 = PublishedImage( + registry="docker.io", + repository="amazoncorretto", + tag="21-al2023", + sha="5454cb606e803fce56861fdbc9eab365eaa2ab4f357ceb8c1d56f4f8c8a7bc33", +) diff --git a/airbyte-ci/connectors/base_images/base_images/sanity_checks.py b/airbyte-ci/connectors/base_images/base_images/sanity_checks.py index a88b137a028d..287636cef73c 100644 --- a/airbyte-ci/connectors/base_images/base_images/sanity_checks.py +++ b/airbyte-ci/connectors/base_images/base_images/sanity_checks.py @@ -178,3 +178,19 @@ async def check_user_can_write_dir(container: dagger.Container, user: str, dir_p await container.with_user(user).with_exec(["touch", f"{dir_path}/foo.txt"]) except dagger.ExecError: raise errors.SanityCheckError(f"{dir_path} is not writable by the {user}.") + + +async def check_file_exists(container: dagger.Container, file_path: str): + """Check that a file exists in the container. + + Args: + container (dagger.Container): The container on which the sanity checks should run. + file_path (str): The file path to check. + + Raises: + errors.SanityCheckError: Raised if the file does not exist. + """ + try: + await container.with_exec(["test", "-f", file_path]) + except dagger.ExecError: + raise errors.SanityCheckError(f"{file_path} does not exist.") diff --git a/airbyte-ci/connectors/base_images/base_images/templates/README.md.j2 b/airbyte-ci/connectors/base_images/base_images/templates/README.md.j2 index 89314b1491d5..c5484077a291 100644 --- a/airbyte-ci/connectors/base_images/base_images/templates/README.md.j2 +++ b/airbyte-ci/connectors/base_images/base_images/templates/README.md.j2 @@ -6,7 +6,7 @@ Our connector build pipeline ([`airbyte-ci`](https://github.com/airbytehq/airbyt Our base images are declared in code, using the [Dagger Python SDK](https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/). - [Python base image code declaration](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/base_images/base_images/python/bases.py) -- ~Java base image code declaration~ *TODO* +- [Java base image code declaration](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/base_images/base_images/java/bases.py) ## Where are the Dockerfiles? @@ -79,6 +79,12 @@ poetry run mypy base_images --check-untyped-defs ## CHANGELOG +### 1.4.0 +- Declare a base image for our java connectors. + +### 1.3.1 +- Update the crane image address. The previous address was deleted by the maintainer. + ### 1.2.0 - Improve new version prompt to pick bump type with optional pre-release version. diff --git a/airbyte-ci/connectors/base_images/base_images/utils/dagger.py b/airbyte-ci/connectors/base_images/base_images/utils/dagger.py new file mode 100644 index 000000000000..0a71d9e80416 --- /dev/null +++ b/airbyte-ci/connectors/base_images/base_images/utils/dagger.py @@ -0,0 +1,6 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + + +def sh_dash_c(lines: list[str]) -> list[str]: + """Wrap sequence of commands in shell for safe usage of dagger Container's with_exec method.""" + return ["sh", "-c", " && ".join(["set -o xtrace"] + lines)] diff --git a/airbyte-ci/connectors/base_images/base_images/version_registry.py b/airbyte-ci/connectors/base_images/base_images/version_registry.py index 41337c8006a8..1495c77c327c 100644 --- a/airbyte-ci/connectors/base_images/base_images/version_registry.py +++ b/airbyte-ci/connectors/base_images/base_images/version_registry.py @@ -13,11 +13,12 @@ import semver from base_images import consts, published_image from base_images.bases import AirbyteConnectorBaseImage +from base_images.java.bases import AirbyteJavaConnectorBaseImage from base_images.python.bases import AirbyteManifestOnlyConnectorBaseImage, AirbytePythonConnectorBaseImage from base_images.utils import docker from connector_ops.utils import ConnectorLanguage # type: ignore -MANAGED_BASE_IMAGES = [AirbytePythonConnectorBaseImage] +MANAGED_BASE_IMAGES = [AirbytePythonConnectorBaseImage, AirbyteJavaConnectorBaseImage] @dataclass @@ -270,6 +271,12 @@ async def get_manifest_only_registry( ) +async def get_java_registry( + dagger_client: dagger.Client, docker_credentials: Tuple[str, str], cache_ttl_seconds: int = 0 +) -> VersionRegistry: + return await VersionRegistry.load(AirbyteJavaConnectorBaseImage, dagger_client, docker_credentials, cache_ttl_seconds=cache_ttl_seconds) + + async def get_registry_for_language( dagger_client: dagger.Client, language: ConnectorLanguage, docker_credentials: Tuple[str, str], cache_ttl_seconds: int = 0 ) -> VersionRegistry: @@ -291,6 +298,8 @@ async def get_registry_for_language( return await get_python_registry(dagger_client, docker_credentials, cache_ttl_seconds=cache_ttl_seconds) elif language is ConnectorLanguage.MANIFEST_ONLY: return await get_manifest_only_registry(dagger_client, docker_credentials, cache_ttl_seconds=cache_ttl_seconds) + elif language is ConnectorLanguage.JAVA: + return await get_java_registry(dagger_client, docker_credentials, cache_ttl_seconds=cache_ttl_seconds) else: raise NotImplementedError(f"Registry for language {language} is not implemented yet.") @@ -298,5 +307,6 @@ async def get_registry_for_language( async def get_all_registries(dagger_client: dagger.Client, docker_credentials: Tuple[str, str]) -> List[VersionRegistry]: return [ await get_python_registry(dagger_client, docker_credentials), - # await get_java_registry(dagger_client), + await get_java_registry(dagger_client, docker_credentials), + # await get_manifest_only_registry(dagger_client, docker_credentials), ] diff --git a/airbyte-ci/connectors/base_images/generated/changelogs/airbyte_java_connector_base.json b/airbyte-ci/connectors/base_images/generated/changelogs/airbyte_java_connector_base.json new file mode 100644 index 000000000000..ca9ab3d5008a --- /dev/null +++ b/airbyte-ci/connectors/base_images/generated/changelogs/airbyte_java_connector_base.json @@ -0,0 +1,22 @@ +[ + { + "version": "1.0.0", + "changelog_entry": "Create a base image for our java connectors based on Amazon Corretto.", + "dockerfile_example": "FROM docker.io/amazoncorretto:21-al2023@sha256:5454cb606e803fce56861fdbc9eab365eaa2ab4f357ceb8c1d56f4f8c8a7bc33\nRUN sh -c set -o xtrace && yum update -y --security && yum install -y tar openssl findutils && yum clean all\nENV AIRBYTE_SPEC_CMD=/airbyte/javabase.sh --spec\nENV AIRBYTE_CHECK_CMD=/airbyte/javabase.sh --check\nENV AIRBYTE_DISCOVER_CMD=/airbyte/javabase.sh --discover\nENV AIRBYTE_READ_CMD=/airbyte/javabase.sh --read\nENV AIRBYTE_WRITE_CMD=/airbyte/javabase.sh --write\nENV AIRBYTE_ENTRYPOINT=/airbyte/base.sh" + }, + { + "version": "1.0.0-rc.4", + "changelog_entry": "Bundle yum calls in a single RUN", + "dockerfile_example": "FROM docker.io/amazoncorretto:21-al2023@sha256:5454cb606e803fce56861fdbc9eab365eaa2ab4f357ceb8c1d56f4f8c8a7bc33\nRUN sh -c set -o xtrace && yum update -y --security && yum install -y tar openssl findutils && yum clean all\nENV AIRBYTE_SPEC_CMD=/airbyte/javabase.sh --spec\nENV AIRBYTE_CHECK_CMD=/airbyte/javabase.sh --check\nENV AIRBYTE_DISCOVER_CMD=/airbyte/javabase.sh --discover\nENV AIRBYTE_READ_CMD=/airbyte/javabase.sh --read\nENV AIRBYTE_WRITE_CMD=/airbyte/javabase.sh --write\nENV AIRBYTE_ENTRYPOINT=/airbyte/base.sh" + }, + { + "version": "1.0.0-rc.2", + "changelog_entry": "Set entrypoint to base.sh", + "dockerfile_example": "FROM docker.io/amazoncorretto:21-al2023@sha256:5454cb606e803fce56861fdbc9eab365eaa2ab4f357ceb8c1d56f4f8c8a7bc33\nRUN yum update -y --security\nRUN yum install -y tar openssl findutils\nENV AIRBYTE_SPEC_CMD=/airbyte/javabase.sh --spec\nENV AIRBYTE_CHECK_CMD=/airbyte/javabase.sh --check\nENV AIRBYTE_DISCOVER_CMD=/airbyte/javabase.sh --discover\nENV AIRBYTE_READ_CMD=/airbyte/javabase.sh --read\nENV AIRBYTE_WRITE_CMD=/airbyte/javabase.sh --write\nENV AIRBYTE_ENTRYPOINT=/airbyte/base.sh" + }, + { + "version": "1.0.0-rc.1", + "changelog_entry": "Create a base image for our java connectors.", + "dockerfile_example": "FROM docker.io/amazoncorretto:21-al2023@sha256:5454cb606e803fce56861fdbc9eab365eaa2ab4f357ceb8c1d56f4f8c8a7bc33\nRUN yum update -y --security\nRUN yum install -y tar openssl findutils\nENV AIRBYTE_SPEC_CMD=/airbyte/javabase.sh --spec\nENV AIRBYTE_CHECK_CMD=/airbyte/javabase.sh --check\nENV AIRBYTE_DISCOVER_CMD=/airbyte/javabase.sh --discover\nENV AIRBYTE_READ_CMD=/airbyte/javabase.sh --read\nENV AIRBYTE_WRITE_CMD=/airbyte/javabase.sh --write\nENV AIRBYTE_ENTRYPOINT=/airbyte/base.sh" + } +] diff --git a/airbyte-ci/connectors/base_images/pyproject.toml b/airbyte-ci/connectors/base_images/pyproject.toml index f6afeb14e5cb..6c4e41f34fea 100644 --- a/airbyte-ci/connectors/base_images/pyproject.toml +++ b/airbyte-ci/connectors/base_images/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "airbyte-connectors-base-images" -version = "1.3.1" +version = "1.4.0" description = "This package is used to generate and publish the base images for Airbyte Connectors." authors = ["Augustin Lafanechere <augustin@airbyte.io>"] readme = "README.md" From 8ad277684ecde7e6fc3c7b7840e92d61caf5026d Mon Sep 17 00:00:00 2001 From: Augustin <augustin@airbyte.io> Date: Tue, 17 Dec 2024 18:55:37 +0100 Subject: [PATCH 6/7] airbyte-ci: use the base image to build java connectors (#49832) --- airbyte-ci/connectors/pipelines/README.md | 7 +++-- .../pipelines/dagger/containers/java.py | 28 +++++++++++++------ .../pipelines/pipelines/helpers/utils.py | 14 ++++++++++ .../connectors/pipelines/pyproject.toml | 2 +- 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index c9206c9b9009..297fc7145936 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -853,9 +853,10 @@ airbyte-ci connectors --language=low-code migrate-to-manifest-only ## Changelog | Version | PR | Description | -|---------|------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------| -| 4.46.5 | [#49835](https://github.com/airbytehq/airbyte/pull/49835) | Fix connector language discovery for projects with Kotlin Gradle build scripts. | -| 4.46.4 | [#49462](https://github.com/airbytehq/airbyte/pull/49462) | Support Kotlin Gradle build scripts in connectors. | +| ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| 4.47.0 | [#49832](https://github.com/airbytehq/airbyte/pull/49462) | Build java connectors from the base image declared in `metadata.yaml`. | +| 4.46.5 | [#49835](https://github.com/airbytehq/airbyte/pull/49835) | Fix connector language discovery for projects with Kotlin Gradle build scripts. | +| 4.46.4 | [#49462](https://github.com/airbytehq/airbyte/pull/49462) | Support Kotlin Gradle build scripts in connectors. | | 4.46.3 | [#49465](https://github.com/airbytehq/airbyte/pull/49465) | Fix `--use-local-cdk` on rootless connectors. | | 4.46.2 | [#49136](https://github.com/airbytehq/airbyte/pull/49136) | Fix failed install of python components due to non-root permissions. | | 4.46.1 | [#49146](https://github.com/airbytehq/airbyte/pull/49146) | Update `crane` image address as the one we were using has been deleted by the maintainer. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py index ad7fb7e4bcdf..47bbe7822b21 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py @@ -9,9 +9,10 @@ from pipelines.consts import AMAZONCORRETTO_IMAGE from pipelines.dagger.actions.connector.hooks import finalize_build from pipelines.dagger.actions.connector.normalization import DESTINATION_NORMALIZATION_BUILD_CONFIGURATION, with_normalization -from pipelines.helpers.utils import sh_dash_c +from pipelines.helpers.utils import deprecated, sh_dash_c +@deprecated("This function is deprecated. Please declare an explicit base image to use in the java connector metadata.") def with_integration_base(context: PipelineContext, build_platform: Platform) -> Container: return ( context.dagger_client.container(platform=build_platform) @@ -24,6 +25,7 @@ def with_integration_base(context: PipelineContext, build_platform: Platform) -> ) +@deprecated("This function is deprecated. Please declare an explicit base image to use in the java connector metadata.") def with_integration_base_java(context: PipelineContext, build_platform: Platform) -> Container: integration_base = with_integration_base(context, build_platform) yum_packages_to_install = [ @@ -72,6 +74,7 @@ def with_integration_base_java(context: PipelineContext, build_platform: Platfor ) +@deprecated("This function is deprecated. Please declare an explicit base image to use in the java connector metadata.") def with_integration_base_java_and_normalization(context: ConnectorContext, build_platform: Platform) -> Container: yum_packages_to_install = [ "python3", @@ -158,22 +161,31 @@ async def with_airbyte_java_connector(context: ConnectorContext, connector_java_ ) ) ) - - if ( + # TODO: remove the condition below once all connectors have a base image declared in their metadata. + if "connectorBuildOptions" in context.connector.metadata and "baseImage" in context.connector.metadata["connectorBuildOptions"]: + base_image_address = context.connector.metadata["connectorBuildOptions"]["baseImage"] + context.logger.info(f"Using base image {base_image_address} from connector metadata to build connector.") + base = context.dagger_client.container(platform=build_platform).from_(base_image_address) + elif ( context.connector.supports_normalization and DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["supports_in_connector_normalization"] ): - base = with_integration_base_java_and_normalization(context, build_platform) - entrypoint = ["/airbyte/run_with_normalization.sh"] + context.logger.warn( + f"Connector {context.connector.technical_name} has in-connector normalization enabled. This is supposed to be deprecated. " + f"Please declare a base image address in the connector metadata.yaml file (connectorBuildOptions.baseImage)." + ) + base = with_integration_base_java_and_normalization(context, build_platform).with_entrypoint(["/airbyte/run_with_normalization.sh"]) else: - base = with_integration_base_java(context, build_platform) - entrypoint = ["/airbyte/base.sh"] + context.logger.warn( + f"Connector {context.connector.technical_name} does not declare a base image in its connector metadata. " + f"Please declare a base image address in the connector metadata.yaml file (connectorBuildOptions.baseImage)." + ) + base = with_integration_base_java(context, build_platform).with_entrypoint(["/airbyte/base.sh"]) connector_container = ( base.with_workdir("/airbyte") .with_env_variable("APPLICATION", application) .with_mounted_directory("built_artifacts", build_stage.directory("/airbyte")) .with_exec(sh_dash_c(["mv built_artifacts/* ."])) - .with_entrypoint(entrypoint) ) return await finalize_build(context, connector_container) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py index 8ab32e8754e3..ca397153becd 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py @@ -7,10 +7,12 @@ import contextlib import datetime +import functools import os import re import sys import unicodedata +import warnings import xml.sax.saxutils from io import TextIOWrapper from pathlib import Path @@ -388,3 +390,15 @@ async def raise_if_not_user(container: Container, expected_user: str) -> None: assert ( actual_user == expected_user ), f"Container is not running as the expected user '{expected_user}', it is running as '{actual_user}'." + + +def deprecated(reason: str) -> Callable: + def decorator(func: Callable) -> Callable: + @functools.wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + warnings.warn(f"{func.__name__} is deprecated: {reason}", DeprecationWarning, stacklevel=2) + return func(*args, **kwargs) + + return wrapper + + return decorator diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index dc32ae4df374..468b9b0e925c 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "4.46.5" +version = "4.47.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte <contact@airbyte.io>"] From 9f819a10c7199983c37943990621a36f6771e55d Mon Sep 17 00:00:00 2001 From: Ian Alton <ian.alton@airbyte.io> Date: Tue, 17 Dec 2024 10:37:42 -0800 Subject: [PATCH 7/7] Inline code uses correct color in a table's header row (#49828) --- docusaurus/src/css/custom.css | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docusaurus/src/css/custom.css b/docusaurus/src/css/custom.css index a4e5298b3e57..663f4b360be0 100644 --- a/docusaurus/src/css/custom.css +++ b/docusaurus/src/css/custom.css @@ -261,7 +261,9 @@ table tr:last-child td:last-child { border-bottom-right-radius: 10px; } - +table th code { + color: var(--ifm-color-content); +} table td code { background-color: var(--color-blue-30);